Attic-0.10/0000755000175000017500000000000012272541113012742 5ustar jonasjonas00000000000000Attic-0.10/attic/0000755000175000017500000000000012272541113014046 5ustar jonasjonas00000000000000Attic-0.10/attic/testsuite/0000755000175000017500000000000012272541113016077 5ustar jonasjonas00000000000000Attic-0.10/attic/testsuite/__init__.py0000644000175000017500000000745712223063673020233 0ustar jonasjonas00000000000000import filecmp import os import posix import sys import sysconfig import time import unittest from attic.helpers import st_mtime_ns from attic.xattr import get_all try: import llfuse # Does this version of llfuse support ns precision? have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') except ImportError: have_fuse_mtime_ns = False # The mtime get/set precison varies on different OS and Python versions if 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []): st_mtime_ns_round = 0 elif 'HAVE_UTIMES' in sysconfig.get_config_vars(): st_mtime_ns_round = -6 else: st_mtime_ns_round = -9 has_mtime_ns = sys.version >= '3.3' utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) class AtticTestCase(unittest.TestCase): """ """ assert_in = unittest.TestCase.assertIn assert_not_in = unittest.TestCase.assertNotIn assert_equal = unittest.TestCase.assertEqual assert_not_equal = unittest.TestCase.assertNotEqual assert_raises = unittest.TestCase.assertRaises assert_true = unittest.TestCase.assertTrue def assert_dirs_equal(self, dir1, dir2): diff = filecmp.dircmp(dir1, dir2) self._assert_dirs_equal_cmp(diff) def _assert_dirs_equal_cmp(self, diff): self.assert_equal(diff.left_only, []) self.assert_equal(diff.right_only, []) self.assert_equal(diff.diff_files, []) self.assert_equal(diff.funny_files, []) for filename in diff.common: path1 = os.path.join(diff.left, filename) path2 = os.path.join(diff.right, filename) s1 = os.lstat(path1) s2 = os.lstat(path2) # Assume path2 is on FUSE if st_dev is different fuse = s1.st_dev != s2.st_dev attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev'] if not fuse or not os.path.isdir(path1): # dir nlink is always 1 on our fuse fileystem attrs.append('st_nlink') d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] if not os.path.islink(path1) or utime_supports_fd: # Older versions of llfuse does not support ns precision properly if fuse and not have_fuse_mtime_ns: d1.append(round(st_mtime_ns(s1), -4)) d2.append(round(st_mtime_ns(s2), -4)) d1.append(round(st_mtime_ns(s1), st_mtime_ns_round)) d2.append(round(st_mtime_ns(s2), st_mtime_ns_round)) d1.append(get_all(path1, follow_symlinks=False)) d2.append(get_all(path2, follow_symlinks=False)) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): self._assert_dirs_equal_cmp(sub_diff) def wait_for_mount(self, path, timeout=5): """Wait until a filesystem is mounted on `path` """ timeout += time.time() while timeout > time.time(): if os.path.ismount(path): return time.sleep(.1) raise Exception('wait_for_mount(%s) timeout' % path) def get_tests(suite): """Generates a sequence of tests from a test suite """ for item in suite: try: # TODO: This could be "yield from..." with Python 3.3+ for i in get_tests(item): yield i except TypeError: yield item class TestLoader(unittest.TestLoader): """A customzied test loader that properly detects and filters our test cases """ def loadTestsFromName(self, pattern, module=None): suite = self.discover('attic.testsuite', '*.py') tests = unittest.TestSuite() for test in get_tests(suite): if pattern.lower() in test.id().lower(): tests.addTest(test) return tests Attic-0.10/attic/testsuite/archive.py0000644000175000017500000000152512270021266020075 0ustar jonasjonas00000000000000import msgpack from attic.testsuite import AtticTestCase from attic.archive import ChunkBuffer from attic.key import PlaintextKey class MockCache: def __init__(self): self.objects = {} def add_chunk(self, id, data, stats=None): self.objects[id] = data return id, len(data), len(data) class ChunkBufferTestCase(AtticTestCase): def test(self): data = [{b'foo': 1}, {b'bar': 2}] cache = MockCache() key = PlaintextKey() chunks = ChunkBuffer(cache, key, None) for d in data: chunks.add(d) chunks.flush() chunks.flush(flush=True) self.assert_equal(len(chunks.chunks), 2) unpacker = msgpack.Unpacker() for id in chunks.chunks: unpacker.feed(cache.objects[id]) self.assert_equal(data, list(unpacker)) Attic-0.10/attic/testsuite/archiver.py0000644000175000017500000003010312272537650020264 0ustar jonasjonas00000000000000import os from io import StringIO import stat import subprocess import sys import shutil import tempfile import time import unittest from hashlib import sha256 from attic import xattr from attic.archiver import Archiver from attic.repository import Repository from attic.testsuite import AtticTestCase from attic.crypto import bytes_to_long, num_aes_blocks try: import llfuse has_llfuse = True except ImportError: has_llfuse = False src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..', '..') class changedir: def __init__(self, dir): self.dir = dir def __enter__(self): self.old = os.getcwd() os.chdir(self.dir) def __exit__(self, *args, **kw): os.chdir(self.old) class ArchiverTestCase(AtticTestCase): prefix = '' def setUp(self): self.archiver = Archiver() self.tmpdir = tempfile.mkdtemp() self.repository_path = os.path.join(self.tmpdir, 'repository') self.repository_location = self.prefix + self.repository_path self.input_path = os.path.join(self.tmpdir, 'input') self.output_path = os.path.join(self.tmpdir, 'output') self.keys_path = os.path.join(self.tmpdir, 'keys') self.cache_path = os.path.join(self.tmpdir, 'cache') os.environ['ATTIC_KEYS_DIR'] = self.keys_path os.environ['ATTIC_CACHE_DIR'] = self.cache_path os.mkdir(self.input_path) os.mkdir(self.output_path) os.mkdir(self.keys_path) os.mkdir(self.cache_path) self._old_wd = os.getcwd() os.chdir(self.tmpdir) def tearDown(self): shutil.rmtree(self.tmpdir) os.chdir(self._old_wd) def attic(self, *args, **kw): exit_code = kw.get('exit_code', 0) fork = kw.get('fork', False) if fork: try: output = subprocess.check_output((sys.executable, '-m', 'attic.archiver') + args) ret = 0 except subprocess.CalledProcessError as e: output = e.output ret = e.returncode output = os.fsdecode(output) if ret != exit_code: print(output) self.assert_equal(exit_code, ret) return output args = list(args) stdout, stderr = sys.stdout, sys.stderr try: output = StringIO() sys.stdout = sys.stderr = output ret = self.archiver.run(args) sys.stdout, sys.stderr = stdout, stderr if ret != exit_code: print(output.getvalue()) self.assert_equal(exit_code, ret) return output.getvalue() finally: sys.stdout, sys.stderr = stdout, stderr def create_src_archive(self, name): self.attic('create', self.repository_location + '::' + name, src_dir) def create_regual_file(self, name, size=0): filename = os.path.join(self.input_path, name) if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) with open(filename, 'wb') as fd: fd.write(b'X' * size) def create_test_files(self): """Create a minimal test case including all supported file types """ # File self.create_regual_file('empty', size=0) self.create_regual_file('file1', size=1024 * 80) # Directory self.create_regual_file('dir2/file2', size=1024 * 80) # File owner os.chown('input/file1', 100, 200) # File mode os.chmod('input/file1', 0o7755) os.chmod('input/dir2', 0o555) # Block device os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20)) # Char device os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40)) if xattr.is_enabled(): xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar') # Hard link os.link(os.path.join(self.input_path, 'file1'), os.path.join(self.input_path, 'hardlink')) # Symlink os.symlink('somewhere', os.path.join(self.input_path, 'link1')) # FIFO node os.mkfifo(os.path.join(self.input_path, 'fifo1')) def test_basic_functionality(self): self.create_test_files() self.attic('init', self.repository_location) self.attic('create', self.repository_location + '::test', 'input') self.attic('create', self.repository_location + '::test.2', 'input') with changedir('output'): self.attic('extract', self.repository_location + '::test') self.assert_equal(len(self.attic('list', self.repository_location).splitlines()), 2) self.assert_equal(len(self.attic('list', self.repository_location + '::test').splitlines()), 10) self.assert_dirs_equal('input', 'output/input') info_output = self.attic('info', self.repository_location + '::test') shutil.rmtree(self.cache_path) info_output2 = self.attic('info', self.repository_location + '::test') # info_output2 starts with some "initializing cache" text but should # end the same way as info_output assert info_output2.endswith(info_output) def test_extract_include_exclude(self): self.attic('init', self.repository_location) self.create_regual_file('file1', size=1024 * 80) self.create_regual_file('file2', size=1024 * 80) self.create_regual_file('file3', size=1024 * 80) self.create_regual_file('file4', size=1024 * 80) self.attic('create', '--exclude=input/file4', self.repository_location + '::test', 'input') with changedir('output'): self.attic('extract', self.repository_location + '::test', 'input/file1', ) self.assert_equal(sorted(os.listdir('output/input')), ['file1']) with changedir('output'): self.attic('extract', '--exclude=input/file2', self.repository_location + '::test') self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3']) def test_path_normalization(self): self.attic('init', self.repository_location) self.create_regual_file('dir1/dir2/file', size=1024 * 80) with changedir('input/dir1/dir2'): self.attic('create', self.repository_location + '::test', '../../../input/dir1/../dir1/dir2/..') output = self.attic('list', self.repository_location + '::test') self.assert_not_in('..', output) self.assert_in(' input/dir1/dir2/file', output) def test_overwrite(self): self.create_regual_file('file1', size=1024 * 80) self.create_regual_file('dir2/file2', size=1024 * 80) self.attic('init', self.repository_location) self.attic('create', self.repository_location + '::test', 'input') # Overwriting regular files and directories should be supported os.mkdir('output/input') os.mkdir('output/input/file1') os.mkdir('output/input/dir2') with changedir('output'): self.attic('extract', self.repository_location + '::test') self.assert_dirs_equal('input', 'output/input') # But non-empty dirs should fail os.unlink('output/input/file1') os.mkdir('output/input/file1') os.mkdir('output/input/file1/dir') with changedir('output'): self.attic('extract', self.repository_location + '::test', exit_code=1) def test_delete(self): self.create_regual_file('file1', size=1024 * 80) self.create_regual_file('dir2/file2', size=1024 * 80) self.attic('init', self.repository_location) self.attic('create', self.repository_location + '::test', 'input') self.attic('create', self.repository_location + '::test.2', 'input') self.attic('verify', self.repository_location + '::test') self.attic('verify', self.repository_location + '::test.2') self.attic('delete', self.repository_location + '::test') self.attic('verify', self.repository_location + '::test.2') self.attic('delete', self.repository_location + '::test.2') # Make sure all data except the manifest has been deleted repository = Repository(self.repository_path) self.assert_equal(repository._len(), 1) def test_corrupted_repository(self): self.attic('init', self.repository_location) self.create_src_archive('test') self.attic('verify', self.repository_location + '::test') name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0] fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+') fd.seek(100) fd.write('XXXX') fd.close() self.attic('verify', self.repository_location + '::test', exit_code=1) def test_readonly_repository(self): self.attic('init', self.repository_location) self.create_src_archive('test') os.system('chmod -R ugo-w ' + self.repository_path) try: self.attic('verify', self.repository_location + '::test') finally: # Restore permissions so shutil.rmtree is able to delete it os.system('chmod -R u+w ' + self.repository_path) def test_prune_repository(self): self.attic('init', self.repository_location) self.attic('create', self.repository_location + '::test1', src_dir) self.attic('create', self.repository_location + '::test2', src_dir) self.attic('prune', self.repository_location, '--daily=2') output = self.attic('list', self.repository_location) assert 'test1' not in output assert 'test2' in output def test_usage(self): self.assert_raises(SystemExit, lambda: self.attic()) self.assert_raises(SystemExit, lambda: self.attic('-h')) @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_mount(self): mountpoint = os.path.join(self.tmpdir, 'mountpoint') os.mkdir(mountpoint) self.attic('init', self.repository_location) self.create_test_files() self.attic('create', self.repository_location + '::archive', 'input') try: self.attic('mount', self.repository_location + '::archive', mountpoint, fork=True) self.wait_for_mount(mountpoint) self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input')) finally: if sys.platform.startswith('linux'): os.system('fusermount -u ' + mountpoint) else: os.system('umount ' + mountpoint) os.rmdir(mountpoint) # Give the daemon some time to exit time.sleep(.2) def verify_aes_counter_uniqueness(self, method): seen = set() # Chunks already seen used = set() # counter values already used def verify_uniqueness(): repository = Repository(self.repository_path) for key, _ in repository.index.iteritems(): data = repository.get(key) hash = sha256(data).digest() if not hash in seen: seen.add(hash) num_blocks = num_aes_blocks(len(data) - 41) nonce = bytes_to_long(data[33:41]) for counter in range(nonce, nonce + num_blocks): self.assert_not_in(counter, used) used.add(counter) self.create_test_files() os.environ['ATTIC_PASSPHRASE'] = 'passphrase' self.attic('init', '--encryption=' + method, self.repository_location) verify_uniqueness() self.attic('create', self.repository_location + '::test', 'input') verify_uniqueness() self.attic('create', self.repository_location + '::test.2', 'input') verify_uniqueness() self.attic('delete', self.repository_location + '::test.2') verify_uniqueness() self.assert_equal(used, set(range(len(used)))) def test_aes_counter_uniqueness_keyfile(self): self.verify_aes_counter_uniqueness('keyfile') def test_aes_counter_uniqueness_passphrase(self): self.verify_aes_counter_uniqueness('passphrase') class RemoteArchiverTestCase(ArchiverTestCase): prefix = '__testsuite__:' Attic-0.10/attic/testsuite/chunker.py0000644000175000017500000000431312166627210020116 0ustar jonasjonas00000000000000from attic.chunker import chunkify, buzhash, buzhash_update from attic.testsuite import AtticTestCase from io import BytesIO class ChunkerTestCase(AtticTestCase): def test_chunkify(self): data = b'0' * 1024 * 1024 * 15 + b'Y' parts = [bytes(c) for c in chunkify(BytesIO(data), 2, 0x3, 2, 0)] self.assert_equal(len(parts), 2) self.assert_equal(b''.join(parts), data) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b''), 2, 0x3, 2, 0)], []) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 2, 0x3, 2, 0)], [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz']) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 2, 0x3, 2, 1)], [b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf', b'oobarb', b'oobaz']) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 2, 0x3, 2, 2)], [b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar', b'boobaz']) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 3, 0x3, 3, 0)], [b'foobarboobaz' * 3]) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 3, 0x3, 3, 1)], [b'foobar', b'boo', b'bazfo', b'obar', b'boo', b'bazfo', b'obar', b'boobaz']) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 3, 0x3, 3, 2)], [b'foo', b'barboobaz', b'foo', b'barboobaz', b'foo', b'barboobaz']) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 3, 0x3, 4, 0)], [b'foobarboobaz' * 3]) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 3, 0x3, 4, 1)], [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz']) self.assert_equal([bytes(c) for c in chunkify(BytesIO(b'foobarboobaz' * 3), 3, 0x3, 4, 2)], [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz']) def test_buzhash(self): self.assert_equal(buzhash(b'abcdefghijklmnop', 0), 3795437769) self.assert_equal(buzhash(b'abcdefghijklmnop', 1), 3795400502) self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1)) Attic-0.10/attic/testsuite/crypto.py0000644000175000017500000000335212230302764017775 0ustar jonasjonas00000000000000from binascii import hexlify from attic.testsuite import AtticTestCase from attic.crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, pbkdf2_sha256, get_random_bytes class CryptoTestCase(AtticTestCase): def test_bytes_to_int(self): self.assert_equal(bytes_to_int(b'\0\0\0\1'), 1) def test_bytes_to_long(self): self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1) self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1') def test_pbkdf2_sha256(self): self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 1, 32)), b'120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b') self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 2, 32)), b'ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43') self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 4096, 32)), b'c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a') def test_get_random_bytes(self): bytes = get_random_bytes(10) bytes2 = get_random_bytes(10) self.assert_equal(len(bytes), 10) self.assert_equal(len(bytes2), 10) self.assert_not_equal(bytes, bytes2) def test_aes(self): key = b'X' * 32 data = b'foo' * 10 aes = AES(key) self.assert_equal(bytes_to_long(aes.iv.raw, 8), 0) cdata = aes.encrypt(data) self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466') self.assert_equal(bytes_to_long(aes.iv.raw, 8), 2) self.assert_not_equal(data, aes.decrypt(cdata)) aes.reset(iv=b'\0' * 16) self.assert_equal(data, aes.decrypt(cdata)) Attic-0.10/attic/testsuite/hashindex.py0000644000175000017500000000571312225606136020437 0ustar jonasjonas00000000000000import hashlib import os import tempfile from attic.hashindex import NSIndex, ChunkIndex from attic.testsuite import AtticTestCase class HashIndexTestCase(AtticTestCase): def _generic_test(self, cls, make_value, sha): idx_name = tempfile.NamedTemporaryFile() idx = cls.create(idx_name.name) self.assert_equal(len(idx), 0) # Test set for x in range(100): idx[bytes('%-32d' % x, 'ascii')] = make_value(x) self.assert_equal(len(idx), 100) for x in range(100): self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x)) # Test update for x in range(100): idx[bytes('%-32d' % x, 'ascii')] = make_value(x * 2) self.assert_equal(len(idx), 100) for x in range(100): self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2)) # Test delete for x in range(50): del idx[bytes('%-32d' % x, 'ascii')] self.assert_equal(len(idx), 50) del idx # Verify file contents with open(idx_name.name, 'rb') as fd: self.assert_equal(hashlib.sha256(fd.read()).hexdigest(), sha) # Make sure we can open the file idx = cls(idx_name.name) self.assert_equal(len(idx), 50) for x in range(50, 100): self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2)) idx.clear() self.assert_equal(len(idx), 0) del idx self.assert_equal(len(cls(idx_name.name)), 0) def test_nsindex(self): self._generic_test(NSIndex, lambda x: (x, x), '369a18ae6a52524eb2884a3c0fdc2824947edd017a2688c5d4d7b3510c245ab9') def test_chunkindex(self): self._generic_test(ChunkIndex, lambda x: (x, x, x), 'ed22e8a883400453c0ee79a06c54df72c994a54eeefdc6c0989efdc5ee6d07b7') def test_resize(self): n = 2000 # Must be >= MIN_BUCKETS idx_name = tempfile.NamedTemporaryFile() idx = NSIndex.create(idx_name.name) initial_size = os.path.getsize(idx_name.name) self.assert_equal(len(idx), 0) for x in range(n): idx[bytes('%-32d' % x, 'ascii')] = x, x idx.flush() self.assert_true(initial_size < os.path.getsize(idx_name.name)) for x in range(n): del idx[bytes('%-32d' % x, 'ascii')] self.assert_equal(len(idx), 0) idx.flush() self.assert_equal(initial_size, os.path.getsize(idx_name.name)) def test_read_only(self): """Make sure read_only indices work even they contain a lot of tombstones """ idx_name = tempfile.NamedTemporaryFile() idx = NSIndex.create(idx_name.name) for x in range(100): idx[bytes('%-0.32d' % x, 'ascii')] = x, x for x in range(99): del idx[bytes('%-0.32d' % x, 'ascii')] idx.flush() idx2 = NSIndex(idx_name.name, readonly=True) self.assert_equal(idx2[bytes('%-0.32d' % 99, 'ascii')], (99, 99)) Attic-0.10/attic/testsuite/helpers.py0000644000175000017500000000772112272273162020130 0ustar jonasjonas00000000000000from datetime import datetime import os import tempfile import unittest from attic.helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, UpgradableLock from attic.testsuite import AtticTestCase class LocationTestCase(AtticTestCase): def test(self): self.assert_equal( repr(Location('ssh://user@host:1234/some/path::archive')), "Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive='archive')" ) self.assert_equal( repr(Location('file:///some/path::archive')), "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive')" ) self.assert_equal( repr(Location('user@host:/some/path::archive')), "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive='archive')" ) self.assert_equal( repr(Location('mybackup.attic::archive')), "Location(proto='file', user=None, host=None, port=None, path='mybackup.attic', archive='archive')" ) self.assert_equal( repr(Location('/some/absolute/path::archive')), "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')" ) self.assert_equal( repr(Location('some/relative/path::archive')), "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')" ) class FormatTimedeltaTestCase(AtticTestCase): def test(self): t0 = datetime(2001, 1, 1, 10, 20, 3, 0) t1 = datetime(2001, 1, 1, 12, 20, 4, 100000) self.assert_equal( format_timedelta(t1 - t0), '2 hours 1.10 seconds' ) class PatternTestCase(AtticTestCase): files = [ '/etc/passwd', '/etc/hosts', '/home/user/.profile', '/home/user/.bashrc', '/home/user2/.profile', '/home/user2/public_html/index.html', '/var/log/messages', '/var/log/dmesg', ] def evaluate(self, paths, excludes): patterns = adjust_patterns(paths, [ExcludePattern(p) for p in excludes]) return [path for path in self.files if not exclude_path(path, patterns)] def test(self): self.assert_equal(self.evaluate(['/'], ['/home']), ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']) self.assert_equal(self.evaluate(['/'], ['*.profile', '/var/log']), ['/etc/passwd', '/etc/hosts', '/home/user/.bashrc', '/home/user2/public_html/index.html']) self.assert_equal(self.evaluate(['/'], ['/home/*/public_html', '*.profile', '*/log/*']), ['/etc/passwd', '/etc/hosts', '/home/user/.bashrc']) self.assert_equal(self.evaluate(['/etc', '/var'], ['dmesg']), ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']) class MakePathSafeTestCase(AtticTestCase): def test(self): self.assert_equal(make_path_safe('/foo/bar'), 'foo/bar') self.assert_equal(make_path_safe('/foo/bar'), 'foo/bar') self.assert_equal(make_path_safe('../foo/bar'), 'foo/bar') self.assert_equal(make_path_safe('../../foo/bar'), 'foo/bar') self.assert_equal(make_path_safe('/'), '.') self.assert_equal(make_path_safe('/'), '.') class UpgradableLockTestCase(AtticTestCase): def test(self): file = tempfile.NamedTemporaryFile() lock = UpgradableLock(file.name) lock.upgrade() lock.upgrade() lock.release() @unittest.skipIf(os.getuid() == 0, 'Root can always open files for writing') def test_read_only_lock_file(self): file = tempfile.NamedTemporaryFile() os.chmod(file.name, 0o444) lock = UpgradableLock(file.name) self.assert_raises(UpgradableLock.LockUpgradeFailed, lock.upgrade) lock.release() Attic-0.10/attic/testsuite/key.py0000644000175000017500000001171212202152326017240 0ustar jonasjonas00000000000000import os import re import shutil import tempfile from binascii import hexlify from attic.crypto import bytes_to_long, num_aes_blocks from attic.testsuite import AtticTestCase from attic.key import PlaintextKey, PassphraseKey, KeyfileKey from attic.helpers import Location, unhexlify class KeyTestCase(AtticTestCase): class MockArgs(object): repository = Location(tempfile.mkstemp()[1]) keyfile2_key_file = """ ATTIC KEY 0000000000000000000000000000000000000000000000000000000000000000 hqppdGVyYXRpb25zzgABhqCkaGFzaNoAIMyonNI+7Cjv0qHi0AOBM6bLGxACJhfgzVD2oq bIS9SFqWFsZ29yaXRobaZzaGEyNTakc2FsdNoAINNK5qqJc1JWSUjACwFEWGTdM7Nd0a5l 1uBGPEb+9XM9p3ZlcnNpb24BpGRhdGHaANAYDT5yfPpU099oBJwMomsxouKyx/OG4QIXK2 hQCG2L2L/9PUu4WIuKvGrsXoP7syemujNfcZws5jLp2UPva4PkQhQsrF1RYDEMLh2eF9Ol rwtkThq1tnh7KjWMG9Ijt7/aoQtq0zDYP/xaFF8XXSJxiyP5zjH5+spB6RL0oQHvbsliSh /cXJq7jrqmrJ1phd6dg4SHAM/i+hubadZoS6m25OQzYAW09wZD/phG8OVa698Z5ed3HTaT SmrtgJL3EoOKgUI9d6BLE4dJdBqntifo""".strip() keyfile2_cdata = unhexlify(re.sub('\W', '', """ 0055f161493fcfc16276e8c31493c4641e1eb19a79d0326fad0291e5a9c98e5933 00000000000003e8d21eaf9b86c297a8cd56432e1915bb """)) keyfile2_id = unhexlify('c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314') def setUp(self): self.tmppath = tempfile.mkdtemp() os.environ['ATTIC_KEYS_DIR'] = self.tmppath def tearDown(self): shutil.rmtree(self.tmppath) class MockRepository(object): class _Location(object): orig = '/some/place' _location = _Location() id = bytes(32) def test_plaintext(self): key = PlaintextKey.create(None, None) data = b'foo' self.assert_equal(hexlify(key.id_hash(data)), b'2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae') self.assert_equal(data, key.decrypt(key.id_hash(data), key.encrypt(data))) def test_keyfile(self): os.environ['ATTIC_PASSPHRASE'] = 'test' key = KeyfileKey.create(self.MockRepository(), self.MockArgs()) self.assert_equal(bytes_to_long(key.enc_cipher.iv, 8), 0) manifest = key.encrypt(b'XXX') self.assert_equal(key.extract_nonce(manifest), 0) manifest2 = key.encrypt(b'XXX') self.assert_not_equal(manifest, manifest2) self.assert_equal(key.decrypt(None, manifest), key.decrypt(None, manifest2)) self.assert_equal(key.extract_nonce(manifest2), 1) iv = key.extract_nonce(manifest) key2 = KeyfileKey.detect(self.MockRepository(), manifest) self.assert_equal(bytes_to_long(key2.enc_cipher.iv, 8), iv + num_aes_blocks(len(manifest) - KeyfileKey.PAYLOAD_OVERHEAD)) # Key data sanity check self.assert_equal(len(set([key2.id_key, key2.enc_key, key2.enc_hmac_key])), 3) self.assert_equal(key2.chunk_seed == 0, False) data = b'foo' self.assert_equal(data, key2.decrypt(key.id_hash(data), key.encrypt(data))) def test_keyfile2(self): with open(os.path.join(os.environ['ATTIC_KEYS_DIR'], 'keyfile'), 'w') as fd: fd.write(self.keyfile2_key_file) os.environ['ATTIC_PASSPHRASE'] = 'passphrase' key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata) self.assert_equal(key.decrypt(self.keyfile2_id, self.keyfile2_cdata), b'payload') def test_passphrase(self): os.environ['ATTIC_PASSPHRASE'] = 'test' key = PassphraseKey.create(self.MockRepository(), None) self.assert_equal(bytes_to_long(key.enc_cipher.iv, 8), 0) self.assert_equal(hexlify(key.id_key), b'793b0717f9d8fb01c751a487e9b827897ceea62409870600013fbc6b4d8d7ca6') self.assert_equal(hexlify(key.enc_hmac_key), b'b885a05d329a086627412a6142aaeb9f6c54ab7950f996dd65587251f6bc0901') self.assert_equal(hexlify(key.enc_key), b'2ff3654c6daf7381dbbe718d2b20b4f1ea1e34caa6cc65f6bb3ac376b93fed2a') self.assert_equal(key.chunk_seed, -775740477) manifest = key.encrypt(b'XXX') self.assert_equal(key.extract_nonce(manifest), 0) manifest2 = key.encrypt(b'XXX') self.assert_not_equal(manifest, manifest2) self.assert_equal(key.decrypt(None, manifest), key.decrypt(None, manifest2)) self.assert_equal(key.extract_nonce(manifest2), 1) iv = key.extract_nonce(manifest) key2 = PassphraseKey.detect(self.MockRepository(), manifest) self.assert_equal(bytes_to_long(key2.enc_cipher.iv, 8), iv + num_aes_blocks(len(manifest) - PassphraseKey.PAYLOAD_OVERHEAD)) self.assert_equal(key.id_key, key2.id_key) self.assert_equal(key.enc_hmac_key, key2.enc_hmac_key) self.assert_equal(key.enc_key, key2.enc_key) self.assert_equal(key.chunk_seed, key2.chunk_seed) data = b'foo' self.assert_equal(hexlify(key.id_hash(data)), b'818217cf07d37efad3860766dcdf1d21e401650fed2d76ed1d797d3aae925990') self.assert_equal(data, key2.decrypt(key2.id_hash(data), key.encrypt(data))) Attic-0.10/attic/testsuite/lrucache.py0000644000175000017500000000231212267034244020243 0ustar jonasjonas00000000000000from attic.lrucache import LRUCache from attic.testsuite import AtticTestCase class LRUCacheTestCase(AtticTestCase): def test(self): c = LRUCache(2) self.assert_equal(len(c), 0) for i, x in enumerate('abc'): c[x] = i self.assert_equal(len(c), 2) self.assert_equal(set(c), set(['b', 'c'])) self.assert_equal(set(c.items()), set([('b', 1), ('c', 2)])) self.assert_equal(False, 'a' in c) self.assert_equal(True, 'b' in c) self.assert_raises(KeyError, lambda: c['a']) self.assert_equal(c['b'], 1) self.assert_equal(c['c'], 2) c['d'] = 3 self.assert_equal(len(c), 2) self.assert_equal(c['c'], 2) self.assert_equal(c['d'], 3) c['c'] = 22 c['e'] = 4 self.assert_equal(len(c), 2) self.assert_raises(KeyError, lambda: c['d']) self.assert_equal(c['c'], 22) self.assert_equal(c['e'], 4) del c['c'] self.assert_equal(len(c), 1) self.assert_raises(KeyError, lambda: c['c']) self.assert_equal(c['e'], 4) def test_pop(self): c = LRUCache(2) c[1] = 1 c[2] = 2 c.pop(1) c[3] = 3 Attic-0.10/attic/testsuite/repository.py0000644000175000017500000001100412225606152020667 0ustar jonasjonas00000000000000import os import shutil import tempfile from attic.hashindex import NSIndex from attic.helpers import Location from attic.remote import RemoteRepository from attic.repository import Repository from attic.testsuite import AtticTestCase class RepositoryTestCase(AtticTestCase): def open(self, create=False): return Repository(os.path.join(self.tmppath, 'repository'), create=create) def setUp(self): self.tmppath = tempfile.mkdtemp() self.repository = self.open(create=True) def tearDown(self): self.repository.close() shutil.rmtree(self.tmppath) def test1(self): for x in range(100): self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') key50 = ('%-32d' % 50).encode('ascii') self.assert_equal(self.repository.get(key50), b'SOMEDATA') self.repository.delete(key50) self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(key50)) self.repository.commit() self.repository.close() repository2 = self.open() self.assert_raises(Repository.DoesNotExist, lambda: repository2.get(key50)) for x in range(100): if x == 50: continue self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA') repository2.close() def test2(self): """Test multiple sequential transactions """ self.repository.put(b'00000000000000000000000000000000', b'foo') self.repository.put(b'00000000000000000000000000000001', b'foo') self.repository.commit() self.repository.delete(b'00000000000000000000000000000000') self.repository.put(b'00000000000000000000000000000001', b'bar') self.repository.commit() self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar') def test_index_rebuild(self): """Verify that repository index rebuild works properly """ def extract_and_unlink_index(): index_name = [n for n in os.listdir(os.path.join(self.tmppath, 'repository')) if n.startswith('index')][0] idx = NSIndex(os.path.join(self.tmppath, 'repository', index_name)) os.unlink(os.path.join(self.tmppath, 'repository', index_name)) return list(idx.iteritems()) self.test2() self.repository.close() before = extract_and_unlink_index() self.open() self.assert_equal(before, extract_and_unlink_index()) def test_consistency(self): """Test cache consistency """ self.repository.put(b'00000000000000000000000000000000', b'foo') self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') self.repository.put(b'00000000000000000000000000000000', b'foo2') self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') self.repository.put(b'00000000000000000000000000000000', b'bar') self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar') self.repository.delete(b'00000000000000000000000000000000') self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(b'00000000000000000000000000000000')) def test_consistency2(self): """Test cache consistency2 """ self.repository.put(b'00000000000000000000000000000000', b'foo') self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') self.repository.commit() self.repository.put(b'00000000000000000000000000000000', b'foo2') self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') self.repository.rollback() self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') def test_single_kind_transactions(self): # put self.repository.put(b'00000000000000000000000000000000', b'foo') self.repository.commit() self.repository.close() # replace self.repository = self.open() self.repository.put(b'00000000000000000000000000000000', b'bar') self.repository.commit() self.repository.close() # delete self.repository = self.open() self.repository.delete(b'00000000000000000000000000000000') self.repository.commit() class RemoteRepositoryTestCase(RepositoryTestCase): def open(self, create=False): return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create) Attic-0.10/attic/testsuite/run.py0000644000175000017500000000025012166625362017265 0ustar jonasjonas00000000000000import unittest from attic.testsuite import TestLoader def main(): unittest.main(testLoader=TestLoader(), defaultTest='') if __name__ == '__main__': main() Attic-0.10/attic/testsuite/xattr.py0000644000175000017500000000261112175026076017623 0ustar jonasjonas00000000000000import os import tempfile import unittest from attic.testsuite import AtticTestCase from attic.xattr import is_enabled, getxattr, setxattr, listxattr @unittest.skipUnless(is_enabled(), 'xattr not enabled on filesystem') class XattrTestCase(AtticTestCase): def setUp(self): self.tmpfile = tempfile.NamedTemporaryFile() self.symlink = os.path.join(os.path.dirname(self.tmpfile.name), 'symlink') os.symlink(self.tmpfile.name, self.symlink) def tearDown(self): os.unlink(self.symlink) def test(self): self.assert_equal(listxattr(self.tmpfile.name), []) self.assert_equal(listxattr(self.tmpfile.fileno()), []) self.assert_equal(listxattr(self.symlink), []) setxattr(self.tmpfile.name, 'user.foo', b'bar') setxattr(self.tmpfile.fileno(), 'user.bar', b'foo') self.assert_equal(set(listxattr(self.tmpfile.name)), set(['user.foo', 'user.bar'])) self.assert_equal(set(listxattr(self.tmpfile.fileno())), set(['user.foo', 'user.bar'])) self.assert_equal(set(listxattr(self.symlink)), set(['user.foo', 'user.bar'])) self.assert_equal(listxattr(self.symlink, follow_symlinks=False), []) self.assert_equal(getxattr(self.tmpfile.name, 'user.foo'), b'bar') self.assert_equal(getxattr(self.tmpfile.fileno(), 'user.foo'), b'bar') self.assert_equal(getxattr(self.symlink, 'user.foo'), b'bar') Attic-0.10/attic/__init__.py0000644000175000017500000000017012200006626016151 0ustar jonasjonas00000000000000# This is a python package from ._version import get_versions __version__ = get_versions()['version'] del get_versions Attic-0.10/attic/archive.py0000644000175000017500000003476612272537255016075 0ustar jonasjonas00000000000000from datetime import datetime, timedelta, timezone from getpass import getuser from itertools import zip_longest import msgpack import os import socket import stat import sys import time from io import BytesIO from attic import xattr from attic.chunker import chunkify from attic.helpers import Error, uid2user, user2uid, gid2group, group2gid, \ Statistics, decode_dict, st_mtime_ns, make_path_safe ITEMS_BUFFER = 1024 * 1024 CHUNK_MIN = 1024 WINDOW_SIZE = 0xfff CHUNK_MASK = 0xffff utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) has_mtime_ns = sys.version >= '3.3' has_lchmod = hasattr(os, 'lchmod') class DownloadPipeline: def __init__(self, repository, key): self.repository = repository self.key = key def unpack_many(self, ids, filter=None, preload=False): unpacker = msgpack.Unpacker(use_list=False) for data in self.fetch_many(ids): unpacker.feed(data) items = [decode_dict(item, (b'path', b'source', b'user', b'group')) for item in unpacker] if filter: items = [item for item in items if filter(item)] if preload: for item in items: if b'chunks' in item: self.repository.preload([c[0] for c in item[b'chunks']]) for item in items: yield item def fetch_many(self, ids, is_preloaded=False): for id_, data in zip_longest(ids, self.repository.get_many(ids, is_preloaded=is_preloaded)): yield self.key.decrypt(id_, data) class ChunkBuffer: BUFFER_SIZE = 1 * 1024 * 1024 def __init__(self, cache, key, stats): self.buffer = BytesIO() self.packer = msgpack.Packer(unicode_errors='surrogateescape') self.cache = cache self.chunks = [] self.key = key self.stats = stats def add(self, item): self.buffer.write(self.packer.pack(item)) def flush(self, flush=False): if self.buffer.tell() == 0: return self.buffer.seek(0) chunks = list(bytes(s) for s in chunkify(self.buffer, WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed)) self.buffer.seek(0) self.buffer.truncate(0) # Leave the last parital chunk in the buffer unless flush is True end = None if flush or len(chunks) == 1 else -1 for chunk in chunks[:end]: id_, _, _ = self.cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats) self.chunks.append(id_) if end == -1: self.buffer.write(chunks[-1]) def is_full(self): return self.buffer.tell() > self.BUFFER_SIZE class Archive: class DoesNotExist(Error): """Archive {} does not exist""" class AlreadyExists(Error): """Archive {} already exists""" def __init__(self, repository, key, manifest, name, cache=None, create=False, checkpoint_interval=300, numeric_owner=False): self.cwd = os.getcwd() self.key = key self.repository = repository self.cache = cache self.manifest = manifest self.hard_links = {} self.stats = Statistics() self.name = name self.checkpoint_interval = checkpoint_interval self.numeric_owner = numeric_owner self.items_buffer = ChunkBuffer(self.cache, self.key, self.stats) self.pipeline = DownloadPipeline(self.repository, self.key) if create: if name in manifest.archives: raise self.AlreadyExists(name) self.last_checkpoint = time.time() i = 0 while True: self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '') if not self.checkpoint_name in manifest.archives: break i += 1 else: if name not in self.manifest.archives: raise self.DoesNotExist(name) info = self.manifest.archives[name] self.load(info[b'id']) def load(self, id): self.id = id data = self.key.decrypt(self.id, self.repository.get(self.id)) self.metadata = msgpack.unpackb(data) if self.metadata[b'version'] != 1: raise Exception('Unknown archive metadata version') decode_dict(self.metadata, (b'name', b'hostname', b'username', b'time')) self.metadata[b'cmdline'] = [arg.decode('utf-8', 'surrogateescape') for arg in self.metadata[b'cmdline']] self.name = self.metadata[b'name'] @property def ts(self): """Timestamp of archive creation in UTC""" t, f = self.metadata[b'time'].split('.', 1) return datetime.strptime(t, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) + timedelta(seconds=float('.' + f)) def __repr__(self): return 'Archive(%r)' % self.name def iter_items(self, filter=None, preload=False): for item in self.pipeline.unpack_many(self.metadata[b'items'], filter=filter, preload=preload): yield item def add_item(self, item): self.items_buffer.add(item) now = time.time() if now - self.last_checkpoint > self.checkpoint_interval: self.last_checkpoint = now self.write_checkpoint() if self.items_buffer.is_full(): self.items_buffer.flush() def write_checkpoint(self): self.save(self.checkpoint_name) del self.manifest.archives[self.checkpoint_name] self.cache.chunk_decref(self.id) def save(self, name=None): name = name or self.name if name in self.manifest.archives: raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) metadata = { 'version': 1, 'name': name, 'items': self.items_buffer.chunks, 'cmdline': sys.argv, 'hostname': socket.gethostname(), 'username': getuser(), 'time': datetime.utcnow().isoformat(), } data = msgpack.packb(metadata, unicode_errors='surrogateescape') self.id = self.key.id_hash(data) self.cache.add_chunk(self.id, data, self.stats) self.manifest.archives[name] = {'id': self.id, 'time': metadata['time']} self.manifest.write() self.repository.commit() self.cache.commit() def calc_stats(self, cache): def add(id): count, size, csize = self.cache.chunks[id] stats.update(size, csize, count == 1) self.cache.chunks[id] = count - 1, size, csize def add_file_chunks(chunks): for id, _, _ in chunks: add(id) # This function is a bit evil since it abuses the cache to calculate # the stats. The cache transaction must be rolled back afterwards unpacker = msgpack.Unpacker(use_list=False) cache.begin_txn() stats = Statistics() add(self.id) for id, chunk in zip_longest(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])): add(id) unpacker.feed(self.key.decrypt(id, chunk)) for item in unpacker: if b'chunks' in item: stats.nfiles += 1 add_file_chunks(item[b'chunks']) cache.rollback() return stats def extract_item(self, item, restore_attrs=True): dest = self.cwd if item[b'path'].startswith('/') or item[b'path'].startswith('..'): raise Exception('Path should be relative and local') path = os.path.join(dest, item[b'path']) # Attempt to remove existing files, ignore errors on failure try: st = os.lstat(path) if stat.S_ISDIR(st.st_mode): os.rmdir(path) else: os.unlink(path) except OSError: pass mode = item[b'mode'] if stat.S_ISDIR(mode): if not os.path.exists(path): os.makedirs(path) if restore_attrs: self.restore_attrs(path, item) elif stat.S_ISREG(mode): if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) # Hard link? if b'source' in item: source = os.path.join(dest, item[b'source']) if os.path.exists(path): os.unlink(path) os.link(source, path) else: with open(path, 'wb') as fd: ids = [c[0] for c in item[b'chunks']] for data in self.pipeline.fetch_many(ids, is_preloaded=True): fd.write(data) fd.flush() self.restore_attrs(path, item, fd=fd.fileno()) elif stat.S_ISFIFO(mode): if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) os.mkfifo(path) self.restore_attrs(path, item) elif stat.S_ISLNK(mode): if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) source = item[b'source'] if os.path.exists(path): os.unlink(path) os.symlink(source, path) self.restore_attrs(path, item, symlink=True) elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode): os.mknod(path, item[b'mode'], item[b'rdev']) self.restore_attrs(path, item) else: raise Exception('Unknown archive item type %r' % item[b'mode']) def restore_attrs(self, path, item, symlink=False, fd=None): xattrs = item.get(b'xattrs') if xattrs: for k, v in xattrs.items(): xattr.setxattr(fd or path, k, v) uid = gid = None if not self.numeric_owner: uid = user2uid(item[b'user']) gid = group2gid(item[b'group']) uid = uid or item[b'uid'] gid = gid or item[b'gid'] # This code is a bit of a mess due to os specific differences try: if fd: os.fchown(fd, uid, gid) else: os.lchown(path, uid, gid) except OSError: pass if fd: os.fchmod(fd, item[b'mode']) elif not symlink: os.chmod(path, item[b'mode']) elif has_lchmod: # Not available on Linux os.lchmod(path, item[b'mode']) if fd and utime_supports_fd: # Python >= 3.3 os.utime(fd, None, ns=(item[b'mtime'], item[b'mtime'])) elif utime_supports_fd: # Python >= 3.3 os.utime(path, None, ns=(item[b'mtime'], item[b'mtime']), follow_symlinks=False) elif not symlink: os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9)) def verify_file(self, item, start, result): if not item[b'chunks']: start(item) result(item, True) else: start(item) ids = [id for id, size, csize in item[b'chunks']] try: for _ in self.pipeline.fetch_many(ids, is_preloaded=True): pass except Exception: result(item, False) return result(item, True) def delete(self, cache): unpacker = msgpack.Unpacker(use_list=False) for id_, data in zip_longest(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])): unpacker.feed(self.key.decrypt(id_, data)) self.cache.chunk_decref(id_) for item in unpacker: if b'chunks' in item: for chunk_id, size, csize in item[b'chunks']: self.cache.chunk_decref(chunk_id) self.cache.chunk_decref(self.id) del self.manifest.archives[self.name] self.manifest.write() self.repository.commit() cache.commit() def stat_attrs(self, st, path): item = { b'mode': st.st_mode, b'uid': st.st_uid, b'user': uid2user(st.st_uid), b'gid': st.st_gid, b'group': gid2group(st.st_gid), b'mtime': st_mtime_ns(st), } if self.numeric_owner: item[b'user'] = item[b'group'] = None xattrs = xattr.get_all(path, follow_symlinks=False) if xattrs: item[b'xattrs'] = xattrs return item def process_item(self, path, st): item = {b'path': make_path_safe(path)} item.update(self.stat_attrs(st, path)) self.add_item(item) def process_dev(self, path, st): item = {b'path': make_path_safe(path), b'rdev': st.st_rdev} item.update(self.stat_attrs(st, path)) self.add_item(item) def process_symlink(self, path, st): source = os.readlink(path) item = {b'path': make_path_safe(path), b'source': source} item.update(self.stat_attrs(st, path)) self.add_item(item) def process_file(self, path, st, cache): safe_path = make_path_safe(path) # Is it a hard link? if st.st_nlink > 1: source = self.hard_links.get((st.st_ino, st.st_dev)) if (st.st_ino, st.st_dev) in self.hard_links: item = self.stat_attrs(st, path) item.update({b'path': safe_path, b'source': source}) self.add_item(item) return else: self.hard_links[st.st_ino, st.st_dev] = safe_path path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape')) ids = cache.file_known_and_unchanged(path_hash, st) chunks = None if ids is not None: # Make sure all ids are available for id_ in ids: if not cache.seen_chunk(id_): break else: chunks = [cache.chunk_incref(id_, self.stats) for id_ in ids] # Only chunkify the file if needed if chunks is None: with open(path, 'rb') as fd: chunks = [] for chunk in chunkify(fd, WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed): chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)) cache.memorize_file(path_hash, st, [c[0] for c in chunks]) item = {b'path': safe_path, b'chunks': chunks} item.update(self.stat_attrs(st, path)) self.stats.nfiles += 1 self.add_item(item) @staticmethod def list_archives(repository, key, manifest, cache=None): for name, info in manifest.archives.items(): yield Archive(repository, key, manifest, name, cache=cache) Attic-0.10/attic/archiver.py0000644000175000017500000005465512272537255016256 0ustar jonasjonas00000000000000import argparse from binascii import hexlify from datetime import datetime from operator import attrgetter import os import stat import sys from attic import __version__ from attic.archive import Archive from attic.repository import Repository from attic.cache import Cache from attic.key import key_creator from attic.helpers import Error, location_validator, format_time, \ format_file_mode, ExcludePattern, exclude_path, adjust_patterns, to_localtime, \ get_cache_dir, get_keys_dir, format_timedelta, prune_split, Manifest, remove_surrogates from attic.remote import RepositoryServer, RemoteRepository class Archiver: def __init__(self): self.exit_code = 0 def open_repository(self, location, create=False): if location.proto == 'ssh': repository = RemoteRepository(location, create=create) else: repository = Repository(location.path, create=create) repository._location = location return repository def print_error(self, msg, *args): msg = args and msg % args or msg self.exit_code = 1 print('attic: ' + msg, file=sys.stderr) def print_verbose(self, msg, *args, **kw): if self.verbose: msg = args and msg % args or msg if kw.get('newline', True): print(msg) else: print(msg, end=' ') def do_serve(self): return RepositoryServer().serve() def do_init(self, args): """Initialize an empty repository """ print('Initializing repository at "%s"' % args.repository.orig) repository = self.open_repository(args.repository, create=True) key = key_creator(repository, args) manifest = Manifest() manifest.repository = repository manifest.key = key manifest.write() repository.commit() return self.exit_code def do_change_passphrase(self, args): """Change repository key file passphrase """ repository = self.open_repository(args.repository) manifest, key = Manifest.load(repository) key.change_passphrase() return self.exit_code def do_create(self, args): """Create new archive """ t0 = datetime.now() repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache, create=True, checkpoint_interval=args.checkpoint_interval, numeric_owner=args.numeric_owner) # Add Attic cache dir to inode_skip list skip_inodes = set() try: st = os.stat(get_cache_dir()) skip_inodes.add((st.st_ino, st.st_dev)) except IOError: pass # Add local repository dir to inode_skip list if not args.archive.host: try: st = os.stat(args.archive.path) skip_inodes.add((st.st_ino, st.st_dev)) except IOError: pass for path in args.paths: path = os.path.normpath(path) if args.dontcross: try: restrict_dev = os.lstat(path).st_dev except OSError as e: self.print_error('%s: %s', path, e) continue else: restrict_dev = None self._process(archive, cache, args.excludes, skip_inodes, path, restrict_dev) archive.save() if args.stats: t = datetime.now() diff = t - t0 print('-' * 40) print('Archive name: %s' % args.archive.archive) print('Archive fingerprint: %s' % hexlify(archive.id).decode('ascii')) print('Start time: %s' % t0.strftime('%c')) print('End time: %s' % t.strftime('%c')) print('Duration: %s' % format_timedelta(diff)) archive.stats.print_() print('-' * 40) return self.exit_code def _process(self, archive, cache, excludes, skip_inodes, path, restrict_dev): if exclude_path(path, excludes): return try: st = os.lstat(path) except OSError as e: self.print_error('%s: %s', path, e) return if (st.st_ino, st.st_dev) in skip_inodes: return # Entering a new filesystem? if restrict_dev and st.st_dev != restrict_dev: return # Ignore unix sockets if stat.S_ISSOCK(st.st_mode): return self.print_verbose(remove_surrogates(path)) if stat.S_ISREG(st.st_mode): try: archive.process_file(path, st, cache) except IOError as e: self.print_error('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): archive.process_item(path, st) try: entries = os.listdir(path) except OSError as e: self.print_error('%s: %s', path, e) else: for filename in sorted(entries): self._process(archive, cache, excludes, skip_inodes, os.path.join(path, filename), restrict_dev) elif stat.S_ISLNK(st.st_mode): archive.process_symlink(path, st) elif stat.S_ISFIFO(st.st_mode): archive.process_item(path, st) elif stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode): archive.process_dev(path, st) else: self.print_error('Unknown file type: %s', path) def do_extract(self, args): """Extract archive contents """ repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.archive.archive, numeric_owner=args.numeric_owner) patterns = adjust_patterns(args.paths, args.excludes) dirs = [] for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True): while dirs and not item[b'path'].startswith(dirs[-1][b'path']): archive.extract_item(dirs.pop(-1)) self.print_verbose(remove_surrogates(item[b'path'])) try: if stat.S_ISDIR(item[b'mode']): dirs.append(item) archive.extract_item(item, restore_attrs=False) else: archive.extract_item(item) except IOError as e: self.print_error('%s: %s', remove_surrogates(item[b'path']), e) while dirs: archive.extract_item(dirs.pop(-1)) return self.exit_code def do_delete(self, args): """Delete archive """ repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) archive.delete(cache) return self.exit_code def do_mount(self, args): """Mount archive as a FUSE fileystem """ try: from attic.fuse import AtticOperations except ImportError: self.print_error('the "llfuse" module is required to use this feature') return self.exit_code if not os.path.isdir(args.mountpoint) or not os.access(args.mountpoint, os.R_OK | os.W_OK | os.X_OK): self.print_error('%s: Mountpoint must be a writable directory' % args.mountpoint) return self.exit_code repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) self.print_verbose("Loading archive metadata...", newline=False) archive = Archive(repository, key, manifest, args.archive.archive) self.print_verbose('done') operations = AtticOperations(key, repository, archive) self.print_verbose("Mounting filesystem") try: operations.mount(args.mountpoint, args.options, args.foreground) except RuntimeError: # Relevant error message already printed to stderr by fuse self.exit_code = 1 return self.exit_code def do_list(self, args): """List archive or repository contents """ repository = self.open_repository(args.src) manifest, key = Manifest.load(repository) if args.src.archive: tmap = {1: 'p', 2: 'c', 4: 'd', 6: 'b', 0o10: '-', 0o12: 'l', 0o14: 's'} archive = Archive(repository, key, manifest, args.src.archive) for item in archive.iter_items(): type = tmap.get(item[b'mode'] // 4096, '?') mode = format_file_mode(item[b'mode']) size = 0 if type == '-': try: size = sum(size for _, size, _ in item[b'chunks']) except KeyError: pass mtime = format_time(datetime.fromtimestamp(item[b'mtime'] / 10**9)) if b'source' in item: if type == 'l': extra = ' -> %s' % item[b'source'] else: type = 'h' extra = ' link to %s' % item[b'source'] else: extra = '' print('%s%s %-6s %-6s %8d %s %s%s' % (type, mode, item[b'user'] or item[b'uid'], item[b'group'] or item[b'gid'], size, mtime, remove_surrogates(item[b'path']), extra)) else: for archive in sorted(Archive.list_archives(repository, key, manifest), key=attrgetter('ts')): print('%-20s %s' % (archive.metadata[b'name'], to_localtime(archive.ts).strftime('%c'))) return self.exit_code def do_verify(self, args): """Verify archive consistency """ repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.archive.archive) patterns = adjust_patterns(args.paths, args.excludes) def start_cb(item): self.print_verbose('%s ...', remove_surrogates(item[b'path']), newline=False) def result_cb(item, success): if success: self.print_verbose('OK') else: self.print_verbose('ERROR') self.print_error('%s: verification failed' % remove_surrogates(item[b'path'])) for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True): if stat.S_ISREG(item[b'mode']) and b'chunks' in item: archive.verify_file(item, start_cb, result_cb) return self.exit_code def do_info(self, args): """Show archive details such as disk space used """ repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) stats = archive.calc_stats(cache) print('Name:', archive.name) print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) print('Hostname:', archive.metadata[b'hostname']) print('Username:', archive.metadata[b'username']) print('Time: %s' % to_localtime(archive.ts).strftime('%c')) print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) stats.print_() return self.exit_code def do_prune(self, args): """Prune repository archives according to specified rules """ repository = self.open_repository(args.repository) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest) archives = list(sorted(Archive.list_archives(repository, key, manifest, cache), key=attrgetter('ts'), reverse=True)) if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0: self.print_error('At least one of the "hourly", "daily", "weekly", "monthly" or "yearly" ' 'settings must be specified') return 1 if args.prefix: archives = [archive for archive in archives if archive.name.startswith(args.prefix)] keep = [] if args.hourly: keep += prune_split(archives, '%Y-%m-%d %H', args.hourly) if args.daily: keep += prune_split(archives, '%Y-%m-%d', args.daily, keep) if args.weekly: keep += prune_split(archives, '%G-%V', args.weekly, keep) if args.monthly: keep += prune_split(archives, '%Y-%m', args.monthly, keep) if args.yearly: keep += prune_split(archives, '%Y', args.yearly, keep) keep.sort(key=attrgetter('ts'), reverse=True) to_delete = [a for a in archives if a not in keep] for archive in keep: self.print_verbose('Keeping archive "%s"' % archive.name) for archive in to_delete: self.print_verbose('Pruning archive "%s"', archive.name) archive.delete(cache) return self.exit_code def run(self, args=None): keys_dir = get_keys_dir() if not os.path.exists(keys_dir): os.makedirs(keys_dir) os.chmod(keys_dir, stat.S_IRWXU) cache_dir = get_cache_dir() if not os.path.exists(cache_dir): os.makedirs(cache_dir) os.chmod(cache_dir, stat.S_IRWXU) common_parser = argparse.ArgumentParser(add_help=False) common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='verbose output') # We can't use argpase for "serve" since we don't want it to show up in "Available commands" if args and args[0] == 'serve': return self.do_serve() parser = argparse.ArgumentParser(description='Attic %s - Deduplicated Backups' % __version__) subparsers = parser.add_subparsers(title='Available commands') subparser = subparsers.add_parser('init', parents=[common_parser], description=self.do_init.__doc__) subparser.set_defaults(func=self.do_init) subparser.add_argument('repository', metavar='REPOSITORY', type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', dest='encryption', choices=('none', 'passphrase', 'keyfile'), default='none', help='select encryption method') subparser = subparsers.add_parser('change-passphrase', parents=[common_parser], description=self.do_change_passphrase.__doc__) subparser.set_defaults(func=self.do_change_passphrase) subparser.add_argument('repository', metavar='REPOSITORY', type=location_validator(archive=False)) subparser = subparsers.add_parser('create', parents=[common_parser], description=self.do_create.__doc__) subparser.set_defaults(func=self.do_create) subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the created archive') subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') subparser.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval', type=int, default=300, metavar='SECONDS', help='write checkpoint every SECONDS seconds (Default: 300)') subparser.add_argument('--do-not-cross-mountpoints', dest='dontcross', action='store_true', default=False, help='do not cross mount points') subparser.add_argument('--numeric-owner', dest='numeric_owner', action='store_true', default=False, help='only store numeric user and group identifiers') subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to create') subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, help='paths to archive') subparser = subparsers.add_parser('extract', parents=[common_parser], description=self.do_extract.__doc__) subparser.set_defaults(func=self.do_extract) subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') subparser.add_argument('--numeric-owner', dest='numeric_owner', action='store_true', default=False, help='only obey numeric user and group identifiers') subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to extract') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to extract') subparser = subparsers.add_parser('delete', parents=[common_parser], description=self.do_delete.__doc__) subparser.set_defaults(func=self.do_delete) subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to delete') subparser = subparsers.add_parser('list', parents=[common_parser], description=self.do_list.__doc__) subparser.set_defaults(func=self.do_list) subparser.add_argument('src', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), help='repository/archive to list contents of') subparser = subparsers.add_parser('mount', parents=[common_parser], description=self.do_mount.__doc__) subparser.set_defaults(func=self.do_mount) subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to mount') subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, help='where to mount filesystem') subparser.add_argument('-f', '--foreground', dest='foreground', action='store_true', default=False, help='stay in foreground, do not daemonize') subparser.add_argument('-o', dest='options', type=str, help='Extra mount options') subparser = subparsers.add_parser('verify', parents=[common_parser], description=self.do_verify.__doc__) subparser.set_defaults(func=self.do_verify) subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to verity integrity of') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to verify') subparser = subparsers.add_parser('info', parents=[common_parser], description=self.do_info.__doc__) subparser.set_defaults(func=self.do_info) subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to display information about') subparser = subparsers.add_parser('prune', parents=[common_parser], description=self.do_prune.__doc__) subparser.set_defaults(func=self.do_prune) subparser.add_argument('-H', '--hourly', dest='hourly', type=int, default=0, help='number of hourly archives to keep') subparser.add_argument('-d', '--daily', dest='daily', type=int, default=0, help='number of daily archives to keep') subparser.add_argument('-w', '--weekly', dest='weekly', type=int, default=0, help='number of daily archives to keep') subparser.add_argument('-m', '--monthly', dest='monthly', type=int, default=0, help='number of monthly archives to keep') subparser.add_argument('-y', '--yearly', dest='yearly', type=int, default=0, help='number of yearly archives to keep') subparser.add_argument('-p', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') subparser.add_argument('repository', metavar='REPOSITORY', type=location_validator(archive=False), help='repository to prune') args = parser.parse_args(args or ['-h']) self.verbose = args.verbose return args.func(args) def main(): archiver = Archiver() try: exit_code = archiver.run(sys.argv[1:]) except Error as e: archiver.print_error(e.get_message()) exit_code = e.exit_code except KeyboardInterrupt: archiver.print_error('Error: Keyboard interrupt') exit_code = 1 else: if exit_code: archiver.print_error('Exiting with failure status due to previous errors') sys.exit(exit_code) if __name__ == '__main__': main() Attic-0.10/attic/cache.py0000644000175000017500000002066312270024604015471 0ustar jonasjonas00000000000000from configparser import RawConfigParser from itertools import zip_longest import msgpack import os from binascii import hexlify import shutil from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, UpgradableLock from .hashindex import ChunkIndex class Cache(object): """Client Side cache """ class RepositoryReplay(Error): """Cache is newer than repository, refusing to continue""" def __init__(self, repository, key, manifest): self.timestamp = None self.txn_active = False self.repository = repository self.key = key self.manifest = manifest self.path = os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii')) if not os.path.exists(self.path): self.create() self.open() if self.manifest.id != self.manifest_id: # If repository is older than the cache something fishy is going on if self.timestamp and self.timestamp > manifest.timestamp: raise self.RepositoryReplay() self.sync() self.commit() def __del__(self): self.close() def create(self): """Create a new empty cache at `path` """ os.makedirs(self.path) with open(os.path.join(self.path, 'README'), 'w') as fd: fd.write('This is an Attic cache') config = RawConfigParser() config.add_section('cache') config.set('cache', 'version', '1') config.set('cache', 'repository', hexlify(self.repository.id).decode('ascii')) config.set('cache', 'manifest', '') with open(os.path.join(self.path, 'config'), 'w') as fd: config.write(fd) ChunkIndex.create(os.path.join(self.path, 'chunks').encode('utf-8')) with open(os.path.join(self.path, 'files'), 'w') as fd: pass # empty file def open(self): if not os.path.isdir(self.path): raise Exception('%s Does not look like an Attic cache' % self.path) self.lock = UpgradableLock(os.path.join(self.path, 'config'), exclusive=True) self.rollback() self.config = RawConfigParser() self.config.read(os.path.join(self.path, 'config')) if self.config.getint('cache', 'version') != 1: raise Exception('%s Does not look like an Attic cache') self.id = self.config.get('cache', 'repository') self.manifest_id = unhexlify(self.config.get('cache', 'manifest')) self.timestamp = self.config.get('cache', 'timestamp', fallback=None) self.chunks = ChunkIndex(os.path.join(self.path, 'chunks').encode('utf-8')) self.files = None def close(self): self.lock.release() def _read_files(self): self.files = {} self._newest_mtime = 0 with open(os.path.join(self.path, 'files'), 'rb') as fd: u = msgpack.Unpacker(use_list=True) while True: data = fd.read(64 * 1024) if not data: break u.feed(data) for hash, item in u: item[0] += 1 self.files[hash] = item def begin_txn(self): # Initialize transaction snapshot txn_dir = os.path.join(self.path, 'txn.tmp') os.mkdir(txn_dir) shutil.copy(os.path.join(self.path, 'config'), txn_dir) shutil.copy(os.path.join(self.path, 'chunks'), txn_dir) shutil.copy(os.path.join(self.path, 'files'), txn_dir) os.rename(os.path.join(self.path, 'txn.tmp'), os.path.join(self.path, 'txn.active')) self.txn_active = True def commit(self): """Commit transaction """ if not self.txn_active: return if self.files is not None: with open(os.path.join(self.path, 'files'), 'wb') as fd: for item in self.files.items(): # Discard cached files with the newest mtime to avoid # issues with filesystem snapshots and mtime precision if item[1][0] < 10 and item[1][3] < self._newest_mtime: msgpack.pack(item, fd) self.config.set('cache', 'manifest', hexlify(self.manifest.id).decode('ascii')) self.config.set('cache', 'timestamp', self.manifest.timestamp) with open(os.path.join(self.path, 'config'), 'w') as fd: self.config.write(fd) self.chunks.flush() os.rename(os.path.join(self.path, 'txn.active'), os.path.join(self.path, 'txn.tmp')) shutil.rmtree(os.path.join(self.path, 'txn.tmp')) self.txn_active = False def rollback(self): """Roll back partial and aborted transactions """ # Roll back active transaction txn_dir = os.path.join(self.path, 'txn.active') if os.path.exists(txn_dir): shutil.copy(os.path.join(txn_dir, 'config'), self.path) shutil.copy(os.path.join(txn_dir, 'chunks'), self.path) shutil.copy(os.path.join(txn_dir, 'files'), self.path) os.rename(txn_dir, os.path.join(self.path, 'txn.tmp')) # Remove partial transaction if os.path.exists(os.path.join(self.path, 'txn.tmp')): shutil.rmtree(os.path.join(self.path, 'txn.tmp')) self.txn_active = False def sync(self): """Initializes cache by fetching and reading all archive indicies """ def add(id, size, csize): try: count, size, csize = self.chunks[id] self.chunks[id] = count + 1, size, csize except KeyError: self.chunks[id] = 1, size, csize self.begin_txn() print('Initializing cache...') self.chunks.clear() unpacker = msgpack.Unpacker() for name, info in self.manifest.archives.items(): id = info[b'id'] cdata = self.repository.get(id) data = self.key.decrypt(id, cdata) add(id, len(data), len(cdata)) archive = msgpack.unpackb(data) if archive[b'version'] != 1: raise Exception('Unknown archive metadata version') decode_dict(archive, (b'name', b'hostname', b'username', b'time')) # fixme: argv print('Analyzing archive:', archive[b'name']) for id_, chunk in zip_longest(archive[b'items'], self.repository.get_many(archive[b'items'])): data = self.key.decrypt(id_, chunk) add(id_, len(data), len(chunk)) unpacker.feed(data) for item in unpacker: if b'chunks' in item: for id_, size, csize in item[b'chunks']: add(id_, size, csize) def add_chunk(self, id, data, stats): if not self.txn_active: self.begin_txn() if self.seen_chunk(id): return self.chunk_incref(id, stats) size = len(data) data = self.key.encrypt(data) csize = len(data) self.repository.put(id, data, wait=False) self.chunks[id] = (1, size, csize) stats.update(size, csize, True) return id, size, csize def seen_chunk(self, id): return self.chunks.get(id, (0, 0, 0))[0] def chunk_incref(self, id, stats): if not self.txn_active: self.begin_txn() count, size, csize = self.chunks[id] self.chunks[id] = (count + 1, size, csize) stats.update(size, csize, False) return id, size, csize def chunk_decref(self, id): if not self.txn_active: self.begin_txn() count, size, csize = self.chunks[id] if count == 1: del self.chunks[id] self.repository.delete(id, wait=False) else: self.chunks[id] = (count - 1, size, csize) def file_known_and_unchanged(self, path_hash, st): if self.files is None: self._read_files() entry = self.files.get(path_hash) if (entry and entry[3] == st_mtime_ns(st) and entry[2] == st.st_size and entry[1] == st.st_ino): # reset entry age self.files[path_hash][0] = 0 return entry[4] else: return None def memorize_file(self, path_hash, st, ids): # Entry: Age, inode, size, mtime, chunk ids mtime_ns = st_mtime_ns(st) self.files[path_hash] = 0, st.st_ino, st.st_size, mtime_ns, ids self._newest_mtime = max(self._newest_mtime, mtime_ns) Attic-0.10/attic/chunker.pyx0000644000175000017500000000275212166624326016267 0ustar jonasjonas00000000000000# -*- coding: utf-8 -*- from libc.stdlib cimport free cdef extern from "_chunker.c": ctypedef int uint32_t ctypedef struct Chunker: pass Chunker *chunker_init(object fd, int window_size, int chunk_mask, int min_size, uint32_t seed) void chunker_free(Chunker *chunker) object chunker_process(Chunker *chunker) uint32_t *buzhash_init_table(uint32_t seed) uint32_t c_buzhash "buzhash"(unsigned char *data, size_t len, uint32_t *h) uint32_t c_buzhash_update "buzhash_update"(uint32_t sum, unsigned char remove, unsigned char add, size_t len, uint32_t *h) cdef class chunkify: cdef Chunker *chunker def __cinit__(self, fd, window_size, chunk_mask, min_size, seed): self.chunker = chunker_init(fd, window_size, chunk_mask, min_size, seed & 0xffffffff) def __dealloc__(self): if self.chunker: chunker_free(self.chunker) def __iter__(self): return self def __next__(self): return chunker_process(self.chunker) def buzhash(unsigned char *data, unsigned long seed): cdef uint32_t *table cdef uint32_t sum table = buzhash_init_table(seed & 0xffffffff) sum = c_buzhash(data, len(data), table) free(table) return sum def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, unsigned long seed): cdef uint32_t *table table = buzhash_init_table(seed & 0xffffffff) sum = c_buzhash_update(sum, remove, add, len, table) free(table) return sumAttic-0.10/attic/crypto.py0000644000175000017500000000561612263607675015770 0ustar jonasjonas00000000000000"""A thin ctypes based wrapper for OpenSSL 1.0 """ import os from ctypes import cdll, c_char_p, c_int, c_uint, c_void_p, POINTER, create_string_buffer from ctypes.util import find_library import struct def _find_libcrypto(): _possible_paths = [ find_library('crypto'), os.environ.get('ATTIC_LIBCRYPTO_PATH'), '/usr/local/opt/openssl/lib/libcrypto.dylib', # OS X Brew '/usr/local/lib/libcrypto.so', # FreeBSD Ports '/usr/local/ssl/lib/libcrypto.so' ] for path in _possible_paths: try: lib = cdll.LoadLibrary(path) if hasattr(lib, 'PKCS5_PBKDF2_HMAC'): return lib except OSError: pass raise Exception('Failed to find libcrypto version >= 1.0') libcrypto = _find_libcrypto() libcrypto.PKCS5_PBKDF2_HMAC.argtypes = (c_char_p, c_int, c_char_p, c_int, c_int, c_void_p, c_int, c_char_p) libcrypto.EVP_sha256.restype = c_void_p libcrypto.AES_set_encrypt_key.argtypes = (c_char_p, c_int, c_char_p) libcrypto.AES_ctr128_encrypt.argtypes = (c_char_p, c_char_p, c_int, c_char_p, c_char_p, c_char_p, POINTER(c_uint)) libcrypto.RAND_bytes.argtypes = (c_char_p, c_int) libcrypto.RAND_bytes.restype = c_int _int = struct.Struct('>I') _long = struct.Struct('>Q') bytes_to_int = lambda x, offset=0: _int.unpack_from(x, offset)[0] bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0] long_to_bytes = lambda x: _long.pack(x) def num_aes_blocks(length): """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data """ return (length + 15) // 16 def pbkdf2_sha256(password, salt, iterations, size): """Password based key derivation function 2 (RFC2898) """ key = create_string_buffer(size) rv = libcrypto.PKCS5_PBKDF2_HMAC(password, len(password), salt, len(salt), iterations, libcrypto.EVP_sha256(), size, key) if not rv: raise Exception('PKCS5_PBKDF2_HMAC failed') return key.raw def get_random_bytes(n): """Return n cryptographically strong pseudo-random bytes """ buf = create_string_buffer(n) if libcrypto.RAND_bytes(buf, n) < 1: raise Exception('RAND_bytes failed') return buf.raw class AES: """A thin wrapper around the OpenSSL AES CTR_MODE cipher """ def __init__(self, key, iv=None): self.key = create_string_buffer(2000) self.iv = create_string_buffer(16) self.buf = create_string_buffer(16) self.num = c_uint() self.reset(key, iv) def reset(self, key=None, iv=None): if key: libcrypto.AES_set_encrypt_key(key, len(key) * 8, self.key) if iv: self.iv.raw = iv self.num.value = 0 def encrypt(self, data): out = create_string_buffer(len(data)) libcrypto.AES_ctr128_encrypt(data, out, len(data), self.key, self.iv, self.buf, self.num) return out.raw decrypt = encrypt Attic-0.10/attic/fuse.py0000644000175000017500000001353412270302170015364 0ustar jonasjonas00000000000000from collections import defaultdict import errno import llfuse import os import stat import time from attic.helpers import daemonize # Does this version of llfuse support ns precision? have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') class AtticOperations(llfuse.Operations): """Export Attic archive as a fuse filesystem """ def __init__(self, key, repository, archive): super(AtticOperations, self).__init__() self._inode_count = 0 self.key = key self.repository = repository self.items = {} self.parent = {} self.contents = defaultdict(dict) default_dir = {b'mode': 0o40755, b'mtime': int(time.time() * 1e9), b'uid': os.getuid(), b'gid': os.getgid()} # Loop through all archive items and assign inode numbers and # extract hierarchy information for item in archive.iter_items(): segments = os.fsencode(os.path.normpath(item[b'path'])).split(b'/') num_segments = len(segments) parent = 1 for i, segment in enumerate(segments, 1): # Insert a default root inode if needed if self._inode_count == 0 and segment: self.items[self.allocate_inode()] = default_dir self.parent[1] = 1 # Leaf segment? if i == num_segments: if b'source' in item and stat.S_ISREG(item[b'mode']): inode = self._find_inode(item[b'source']) self.items[inode][b'nlink'] = self.items[inode].get(b'nlink', 1) + 1 else: inode = self.allocate_inode() self.items[inode] = item self.parent[inode] = parent if segment: self.contents[parent][segment] = inode elif segment in self.contents[parent]: parent = self.contents[parent][segment] else: inode = self.allocate_inode() self.items[inode] = default_dir self.parent[inode] = parent if segment: self.contents[parent][segment] = inode parent = inode def allocate_inode(self): self._inode_count += 1 return self._inode_count def statfs(self): stat_ = llfuse.StatvfsData() stat_.f_bsize = 512 stat_.f_frsize = 512 stat_.f_blocks = 0 stat_.f_bfree = 0 stat_.f_bavail = 0 stat_.f_files = 0 stat_.f_ffree = 0 stat_.f_favail = 0 return stat_ def _find_inode(self, path): segments = os.fsencode(os.path.normpath(path)).split(b'/') inode = 1 for segment in segments: inode = self.contents[inode][segment] return inode def getattr(self, inode): item = self.items[inode] size = 0 try: size = sum(size for _, size, _ in item[b'chunks']) except KeyError: pass entry = llfuse.EntryAttributes() entry.st_ino = inode entry.generation = 0 entry.entry_timeout = 300 entry.attr_timeout = 300 entry.st_mode = item[b'mode'] entry.st_nlink = item.get(b'nlink', 1) entry.st_uid = item[b'uid'] entry.st_gid = item[b'gid'] entry.st_rdev = item.get(b'rdev', 0) entry.st_size = size entry.st_blksize = 512 entry.st_blocks = 1 if have_fuse_mtime_ns: entry.st_atime_ns = item[b'mtime'] entry.st_mtime_ns = item[b'mtime'] entry.st_ctime_ns = item[b'mtime'] else: entry.st_atime = item[b'mtime'] / 1e9 entry.st_mtime = item[b'mtime'] / 1e9 entry.st_ctime = item[b'mtime'] / 1e9 return entry def listxattr(self, inode): item = self.items[inode] return item.get(b'xattrs', {}).keys() def getxattr(self, inode, name): item = self.items[inode] try: return item.get(b'xattrs', {})[name] except KeyError: raise llfuse.FUSEError(errno.ENODATA) def lookup(self, parent_inode, name): if name == b'.': inode = parent_inode elif name == b'..': inode = self.parent[parent_inode] else: inode = self.contents[parent_inode].get(name) if not inode: raise llfuse.FUSEError(errno.ENOENT) return self.getattr(inode) def open(self, inode, flags): return inode def opendir(self, inode): return inode def read(self, fh, offset, size): parts = [] item = self.items[fh] for id, s, csize in item[b'chunks']: if s < offset: offset -= s continue n = min(size, s - offset) chunk = self.key.decrypt(id, self.repository.get(id)) parts.append(chunk[offset:offset+n]) offset = 0 size -= n if not size: break return b''.join(parts) def readdir(self, fh, off): entries = [(b'.', fh), (b'..', self.parent[fh])] entries.extend(self.contents[fh].items()) for i, (name, inode) in enumerate(entries[off:], off): yield name, self.getattr(inode), i + 1 def readlink(self, inode): return os.fsencode(self.items[inode][b'source']) def mount(self, mountpoint, extra_options, foreground=False): options = ['fsname=atticfs', 'ro'] if extra_options: options.extend(extra_options.split(',')) llfuse.init(self, mountpoint, options) if not foreground: daemonize() try: llfuse.main(single=True) except: llfuse.close() raise llfuse.close() Attic-0.10/attic/hashindex.pyx0000644000175000017500000001170512272262120016565 0ustar jonasjonas00000000000000# -*- coding: utf-8 -*- import os cdef extern from "_hashindex.c": ctypedef struct HashIndex: pass HashIndex *hashindex_open(char *path, int readonly) HashIndex *hashindex_create(char *path, int capacity, int key_size, int value_size) int hashindex_get_size(HashIndex *index) int hashindex_clear(HashIndex *index) int hashindex_close(HashIndex *index) int hashindex_flush(HashIndex *index) void *hashindex_get(HashIndex *index, void *key) void *hashindex_next_key(HashIndex *index, void *key) int hashindex_delete(HashIndex *index, void *key) int hashindex_set(HashIndex *index, void *key, void *value) int _htole32(int v) int _le32toh(int v) _NoDefault = object() cdef class IndexBase: cdef HashIndex *index key_size = 32 def __cinit__(self, path, readonly=False): self.index = hashindex_open(os.fsencode(path), readonly) if not self.index: raise Exception('Failed to open %s' % path) def __dealloc__(self): if self.index: if not hashindex_close(self.index): raise Exception('hashindex_close failed') @classmethod def create(cls, path): index = hashindex_create(os.fsencode(path), 0, cls.key_size, cls.value_size) if not index: raise Exception('Failed to create %s' % path) hashindex_close(index) return cls(path) def clear(self): if not hashindex_clear(self.index): raise Exception('hashindex_clear failed') def flush(self): if not hashindex_flush(self.index): raise Exception('hashindex_flush failed') def setdefault(self, key, value): if not key in self: self[key] = value def __delitem__(self, key): assert len(key) == 32 if not hashindex_delete(self.index, key): raise Exception('hashindex_delete failed') def get(self, key, default=None): try: return self[key] except KeyError: return default def pop(self, key, default=_NoDefault): try: value = self[key] del self[key] return value except KeyError: if default != _NoDefault: return default raise def __len__(self): return hashindex_get_size(self.index) cdef class NSIndex(IndexBase): value_size = 8 def __getitem__(self, key): assert len(key) == 32 data = hashindex_get(self.index, key) if not data: raise KeyError return _le32toh(data[0]), _le32toh(data[1]) def __setitem__(self, key, value): assert len(key) == 32 cdef int[2] data data[0] = _htole32(value[0]) data[1] = _htole32(value[1]) if not hashindex_set(self.index, key, data): raise Exception('hashindex_set failed') def __contains__(self, key): assert len(key) == 32 data = hashindex_get(self.index, key) return data != NULL def iteritems(self, marker=None, limit=0): iter = NSKeyIterator() iter.index = self.index return iter cdef class NSKeyIterator: cdef HashIndex *index cdef char *key def __cinit__(self): self.key = NULL def __iter__(self): return self def __next__(self): self.key = hashindex_next_key(self.index, self.key) if not self.key: raise StopIteration cdef int *value = (self.key + 32) return self.key[:32], (_le32toh(value[0]), _le32toh(value[1])) cdef class ChunkIndex(IndexBase): value_size = 12 def __getitem__(self, key): assert len(key) == 32 data = hashindex_get(self.index, key) if not data: raise KeyError return _le32toh(data[0]), _le32toh(data[1]), _le32toh(data[2]) def __setitem__(self, key, value): assert len(key) == 32 cdef int[3] data data[0] = _htole32(value[0]) data[1] = _htole32(value[1]) data[2] = _htole32(value[2]) if not hashindex_set(self.index, key, data): raise Exception('hashindex_set failed') def __contains__(self, key): assert len(key) == 32 data = hashindex_get(self.index, key) return data != NULL def iteritems(self, marker=None, limit=0): iter = ChunkKeyIterator() iter.index = self.index return iter cdef class ChunkKeyIterator: cdef HashIndex *index cdef char *key def __cinit__(self): self.key = NULL def __iter__(self): return self def __next__(self): self.key = hashindex_next_key(self.index, self.key) if not self.key: raise StopIteration cdef int *value = (self.key + 32) return self.key[:32], (_le32toh(value[0]), _le32toh(value[1]), _le32toh(value[2])) Attic-0.10/attic/helpers.py0000644000175000017500000002772612272273071016105 0ustar jonasjonas00000000000000import argparse import binascii import grp import msgpack import os import pwd import re import stat import sys import time from datetime import datetime, timezone from fnmatch import fnmatchcase from operator import attrgetter import fcntl class Error(Exception): """Error base class""" exit_code = 1 def get_message(self): return 'Error: ' + type(self).__doc__.format(*self.args) class UpgradableLock: class LockUpgradeFailed(Error): """Failed to acquire write lock on {}""" def __init__(self, path, exclusive=False): self.path = path try: self.fd = open(path, 'r+') except IOError: self.fd = open(path, 'r') if exclusive: fcntl.lockf(self.fd, fcntl.LOCK_EX) else: fcntl.lockf(self.fd, fcntl.LOCK_SH) self.is_exclusive = exclusive def upgrade(self): try: fcntl.lockf(self.fd, fcntl.LOCK_EX) except OSError as e: raise self.LockUpgradeFailed(self.path) self.is_exclusive = True def release(self): fcntl.lockf(self.fd, fcntl.LOCK_UN) self.fd.close() class Manifest: MANIFEST_ID = b'\0' * 32 def __init__(self): self.archives = {} self.config = {} @classmethod def load(cls, repository): from .key import key_factory manifest = cls() manifest.repository = repository cdata = repository.get(manifest.MANIFEST_ID) manifest.key = key = key_factory(repository, cdata) data = key.decrypt(None, cdata) manifest.id = key.id_hash(data) m = msgpack.unpackb(data) if not m.get(b'version') == 1: raise ValueError('Invalid manifest version') manifest.archives = dict((k.decode('utf-8'), v) for k,v in m[b'archives'].items()) manifest.timestamp = m.get(b'timestamp') if manifest.timestamp: manifest.timestamp = manifest.timestamp.decode('ascii') manifest.config = m[b'config'] return manifest, key def write(self): self.timestamp = datetime.utcnow().isoformat() data = msgpack.packb({ 'version': 1, 'archives': self.archives, 'timestamp': self.timestamp, 'config': self.config, }) self.id = self.key.id_hash(data) self.repository.put(self.MANIFEST_ID, self.key.encrypt(data)) def prune_split(archives, pattern, n, skip=[]): items = {} keep = [] for a in archives: key = to_localtime(a.ts).strftime(pattern) items.setdefault(key, []) items[key].append(a) for key, values in sorted(items.items(), reverse=True): if n and values[0] not in skip: values.sort(key=attrgetter('ts'), reverse=True) keep.append(values[0]) n -= 1 return keep class Statistics: def __init__(self): self.osize = self.csize = self.usize = self.nfiles = 0 def update(self, size, csize, unique): self.osize += size self.csize += csize if unique: self.usize += csize def print_(self): print('Number of files: %d' % self.nfiles) print('Original size: %d (%s)' % (self.osize, format_file_size(self.osize))) print('Compressed size: %s (%s)' % (self.csize, format_file_size(self.csize))) print('Unique data: %d (%s)' % (self.usize, format_file_size(self.usize))) def get_keys_dir(): """Determine where to repository keys and cache""" return os.environ.get('ATTIC_KEYS_DIR', os.path.join(os.path.expanduser('~'), '.attic', 'keys')) def get_cache_dir(): """Determine where to repository keys and cache""" return os.environ.get('ATTIC_CACHE_DIR', os.path.join(os.path.expanduser('~'), '.cache', 'attic')) def to_localtime(ts): """Convert datetime object from UTC to local time zone""" return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6]) def adjust_patterns(paths, excludes): if paths: return (excludes or []) + [IncludePattern(path) for path in paths] + [ExcludePattern('*')] else: return excludes def exclude_path(path, patterns): """Used by create and extract sub-commands to determine if an item should be processed or not """ for pattern in (patterns or []): if pattern.match(path): return isinstance(pattern, ExcludePattern) return False class IncludePattern: """--include PATTERN """ def __init__(self, pattern): self.pattern = pattern def match(self, path): dir, name = os.path.split(path) return (path == self.pattern or (dir + os.path.sep).startswith(self.pattern)) def __repr__(self): return '%s(%s)' % (type(self), self.pattern) class ExcludePattern(IncludePattern): """ """ def __init__(self, pattern): self.pattern = self.dirpattern = pattern if not pattern.endswith('/'): self.dirpattern += '/*' def match(self, path): dir, name = os.path.split(path) return fnmatchcase(path, self.pattern) or fnmatchcase(dir + '/', self.dirpattern) def __repr__(self): return '%s(%s)' % (type(self), self.pattern) def walk_path(path, skip_inodes=None): st = os.lstat(path) if skip_inodes and (st.st_ino, st.st_dev) in skip_inodes: return yield path, st if stat.S_ISDIR(st.st_mode): for f in os.listdir(path): for x in walk_path(os.path.join(path, f), skip_inodes): yield x def format_time(t): """Format datetime suitable for fixed length list output """ if (datetime.now() - t).days < 365: return t.strftime('%b %d %H:%M') else: return t.strftime('%b %d %Y') def format_timedelta(td): """Format timedelta in a human friendly format """ # Since td.total_seconds() requires python 2.7 ts = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6) s = ts % 60 m = int(ts / 60) % 60 h = int(ts / 3600) % 24 txt = '%.2f seconds' % s if m: txt = '%d minutes %s' % (m, txt) if h: txt = '%d hours %s' % (h, txt) if td.days: txt = '%d days %s' % (td.days, txt) return txt def format_file_mode(mod): """Format file mode bits for list output """ def x(v): return ''.join(v & m and s or '-' for m, s in ((4, 'r'), (2, 'w'), (1, 'x'))) return '%s%s%s' % (x(mod // 64), x(mod // 8), x(mod)) def format_file_size(v): """Format file size into a human friendly format """ if v > 1024 * 1024 * 1024: return '%.2f GB' % (v / 1024. / 1024. / 1024.) elif v > 1024 * 1024: return '%.2f MB' % (v / 1024. / 1024.) elif v > 1024: return '%.2f kB' % (v / 1024.) else: return '%d B' % v class IntegrityError(Exception): """ """ def memoize(function): cache = {} def decorated_function(*args): try: return cache[args] except KeyError: val = function(*args) cache[args] = val return val return decorated_function @memoize def uid2user(uid): try: return pwd.getpwuid(uid).pw_name except KeyError: return None @memoize def user2uid(user): try: return user and pwd.getpwnam(user).pw_uid except KeyError: return None @memoize def gid2group(gid): try: return grp.getgrgid(gid).gr_name except KeyError: return None @memoize def group2gid(group): try: return group and grp.getgrnam(group).gr_gid except KeyError: return None class Location: """Object representing a repository / archive location """ proto = user = host = port = path = archive = None ssh_re = re.compile(r'(?Pssh)://(?:(?P[^@]+)@)?' r'(?P[^:/#]+)(?::(?P\d+))?' r'(?P[^:]+)(?:::(?P.+))?') file_re = re.compile(r'(?Pfile)://' r'(?P[^:]+)(?:::(?P.+))?') scp_re = re.compile(r'((?:(?P[^@]+)@)?(?P[^:/]+):)?' r'(?P[^:]+)(?:::(?P.+))?') def __init__(self, text): self.orig = text if not self.parse(text): raise ValueError def parse(self, text): m = self.ssh_re.match(text) if m: self.proto = m.group('proto') self.user = m.group('user') self.host = m.group('host') self.port = m.group('port') and int(m.group('port')) or None self.path = m.group('path') self.archive = m.group('archive') return True m = self.file_re.match(text) if m: self.proto = m.group('proto') self.path = m.group('path') self.archive = m.group('archive') return True m = self.scp_re.match(text) if m: self.user = m.group('user') self.host = m.group('host') self.path = m.group('path') self.archive = m.group('archive') self.proto = self.host and 'ssh' or 'file' return True return False def __str__(self): items = [] items.append('proto=%r' % self.proto) items.append('user=%r' % self.user) items.append('host=%r' % self.host) items.append('port=%r' % self.port) items.append('path=%r' % self.path) items.append('archive=%r' % self.archive) return ', '.join(items) def to_key_filename(self): name = re.sub('[^\w]', '_', self.path).strip('_') if self.proto != 'file': name = self.host + '__' + name return os.path.join(get_keys_dir(), name) def __repr__(self): return "Location(%s)" % self def location_validator(archive=None): def validator(text): try: loc = Location(text) except ValueError: raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text) if archive is True and not loc.archive: raise argparse.ArgumentTypeError('"%s": No archive specified' % text) elif archive is False and loc.archive: raise argparse.ArgumentTypeError('"%s" No archive can be specified' % text) return loc return validator def read_msgpack(filename): with open(filename, 'rb') as fd: return msgpack.unpack(fd) def write_msgpack(filename, d): with open(filename + '.tmp', 'wb') as fd: msgpack.pack(d, fd) fd.flush() os.fsync(fd) os.rename(filename + '.tmp', filename) def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'): for key in keys: if isinstance(d.get(key), bytes): d[key] = d[key].decode(encoding, errors) return d def remove_surrogates(s, errors='replace'): """Replace surrogates generated by fsdecode with '?' """ return s.encode('utf-8', errors).decode('utf-8') _safe_re = re.compile('^((..)?/+)+') def make_path_safe(path): """Make path safe by making it relative and local """ return _safe_re.sub('', path) or '.' def daemonize(): """Detach process from controlling terminal and run in background """ pid = os.fork() if pid: os._exit(0) os.setsid() pid = os.fork() if pid: os._exit(0) os.chdir('/') os.close(0) os.close(1) os.close(2) fd = os.open('/dev/null', os.O_RDWR) os.dup2(fd, 0) os.dup2(fd, 1) os.dup2(fd, 2) if sys.version < '3.3': # st_mtime_ns attribute only available in 3.3+ def st_mtime_ns(st): return int(st.st_mtime * 1e9) # unhexlify in < 3.3 incorrectly only accepts bytes input def unhexlify(data): if isinstance(data, str): data = data.encode('ascii') return binascii.unhexlify(data) else: def st_mtime_ns(st): return st.st_mtime_ns unhexlify = binascii.unhexlify Attic-0.10/attic/key.py0000644000175000017500000002570712202152302015212 0ustar jonasjonas00000000000000from binascii import hexlify, a2b_base64, b2a_base64 from getpass import getpass import os import msgpack import textwrap import hmac from hashlib import sha256 import zlib from attic.crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks from attic.helpers import IntegrityError, get_keys_dir PREFIX = b'\0' * 8 class HMAC(hmac.HMAC): """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews """ def update(self, msg): self.inner.update(msg) def key_creator(repository, args): if args.encryption == 'keyfile': return KeyfileKey.create(repository, args) elif args.encryption == 'passphrase': return PassphraseKey.create(repository, args) else: return PlaintextKey.create(repository, args) def key_factory(repository, manifest_data): if manifest_data[0] == KeyfileKey.TYPE: return KeyfileKey.detect(repository, manifest_data) elif manifest_data[0] == PassphraseKey.TYPE: return PassphraseKey.detect(repository, manifest_data) elif manifest_data[0] == PlaintextKey.TYPE: return PlaintextKey.detect(repository, manifest_data) else: raise Exception('Unkown Key type %d' % ord(manifest_data[0])) class KeyBase(object): def __init__(self): self.TYPE_STR = bytes([self.TYPE]) def id_hash(self, data): """Return HMAC hash using the "id" HMAC key """ def encrypt(self, data): pass def decrypt(self, id, data): pass class PlaintextKey(KeyBase): TYPE = 0x02 chunk_seed = 0 @classmethod def create(cls, repository, args): print('Encryption NOT enabled.\nUse the "--encryption=passphrase|keyfile" to enable encryption.') return cls() @classmethod def detect(cls, repository, manifest_data): return cls() def id_hash(self, data): return sha256(data).digest() def encrypt(self, data): return b''.join([self.TYPE_STR, zlib.compress(data)]) def decrypt(self, id, data): if data[0] != self.TYPE: raise IntegrityError('Invalid encryption envelope') data = zlib.decompress(memoryview(data)[1:]) if id and sha256(data).digest() != id: raise IntegrityError('Chunk id verification failed') return data class AESKeyBase(KeyBase): """Common base class shared by KeyfileKey and PassphraseKey Chunks are encrypted using 256bit AES in Counter Mode (CTR) Payload layout: TYPE(1) + HMAC(32) + NONCE(8) + CIPHERTEXT To reduce payload size only 8 bytes of the 16 bytes nonce is saved in the payload, the first 8 bytes are always zeros. This does not affect security but limits the maximum repository capacity to only 295 exabytes! """ PAYLOAD_OVERHEAD = 1 + 32 + 8 # TYPE + HMAC + NONCE def id_hash(self, data): """Return HMAC hash using the "id" HMAC key """ return HMAC(self.id_key, data, sha256).digest() def encrypt(self, data): data = zlib.compress(data) self.enc_cipher.reset() data = b''.join((self.enc_cipher.iv[8:], self.enc_cipher.encrypt(data))) hmac = HMAC(self.enc_hmac_key, data, sha256).digest() return b''.join((self.TYPE_STR, hmac, data)) def decrypt(self, id, data): if data[0] != self.TYPE: raise IntegrityError('Invalid encryption envelope') hmac = memoryview(data)[1:33] if memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest()) != hmac: raise IntegrityError('Encryption envelope checksum mismatch') self.dec_cipher.reset(iv=PREFIX + data[33:41]) data = zlib.decompress(self.dec_cipher.decrypt(data[41:])) # should use memoryview if id and HMAC(self.id_key, data, sha256).digest() != id: raise IntegrityError('Chunk id verification failed') return data def extract_nonce(self, payload): if payload[0] != self.TYPE: raise IntegrityError('Invalid encryption envelope') nonce = bytes_to_long(payload[33:41]) return nonce def init_from_random_data(self, data): self.enc_key = data[0:32] self.enc_hmac_key = data[32:64] self.id_key = data[64:96] self.chunk_seed = bytes_to_int(data[96:100]) # Convert to signed int32 if self.chunk_seed & 0x80000000: self.chunk_seed = self.chunk_seed - 0xffffffff - 1 def init_ciphers(self, enc_iv=b''): self.enc_cipher = AES(self.enc_key, enc_iv) self.dec_cipher = AES(self.enc_key) class PassphraseKey(AESKeyBase): TYPE = 0x01 iterations = 100000 @classmethod def create(cls, repository, args): key = cls() passphrase = os.environ.get('ATTIC_PASSPHRASE') if passphrase is not None: passphrase2 = passphrase else: passphrase, passphrase2 = 1, 2 while passphrase != passphrase2: passphrase = getpass('Enter passphrase: ') if not passphrase: print('Passphrase must not be blank') continue passphrase2 = getpass('Enter same passphrase again: ') if passphrase != passphrase2: print('Passphrases do not match') key.init(repository, passphrase) if passphrase: print('Remember your passphrase. Your data will be inaccessible without it.') return key @classmethod def detect(cls, repository, manifest_data): prompt = 'Enter passphrase for %s: ' % repository._location.orig key = cls() passphrase = os.environ.get('ATTIC_PASSPHRASE') if passphrase is None: passphrase = getpass(prompt) while True: key.init(repository, passphrase) try: key.decrypt(None, manifest_data) num_blocks = num_aes_blocks(len(manifest_data) - 41) key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks)) return key except IntegrityError: passphrase = getpass(prompt) def init(self, repository, passphrase): self.init_from_random_data(pbkdf2_sha256(passphrase.encode('utf-8'), repository.id, self.iterations, 100)) self.init_ciphers() class KeyfileKey(AESKeyBase): FILE_ID = 'ATTIC KEY' TYPE = 0x00 @classmethod def detect(cls, repository, manifest_data): key = cls() path = cls.find_key_file(repository) prompt = 'Enter passphrase for key file %s: ' % path passphrase = os.environ.get('ATTIC_PASSPHRASE', '') while not key.load(path, passphrase): passphrase = getpass(prompt) num_blocks = num_aes_blocks(len(manifest_data) - 41) key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks)) return key @classmethod def find_key_file(cls, repository): id = hexlify(repository.id).decode('ascii') keys_dir = get_keys_dir() for name in os.listdir(keys_dir): filename = os.path.join(keys_dir, name) with open(filename, 'r') as fd: line = fd.readline().strip() if line and line.startswith(cls.FILE_ID) and line[10:] == id: return filename raise Exception('Key file for repository with ID %s not found' % id) def load(self, filename, passphrase): with open(filename, 'r') as fd: cdata = a2b_base64(''.join(fd.readlines()[1:]).encode('ascii')) # .encode needed for Python 3.[0-2] data = self.decrypt_key_file(cdata, passphrase) if data: key = msgpack.unpackb(data) if key[b'version'] != 1: raise IntegrityError('Invalid key file header') self.repository_id = key[b'repository_id'] self.enc_key = key[b'enc_key'] self.enc_hmac_key = key[b'enc_hmac_key'] self.id_key = key[b'id_key'] self.chunk_seed = key[b'chunk_seed'] self.path = filename return True def decrypt_key_file(self, data, passphrase): d = msgpack.unpackb(data) assert d[b'version'] == 1 assert d[b'algorithm'] == b'sha256' key = pbkdf2_sha256(passphrase.encode('utf-8'), d[b'salt'], d[b'iterations'], 32) data = AES(key).decrypt(d[b'data']) if HMAC(key, data, sha256).digest() != d[b'hash']: return None return data def encrypt_key_file(self, data, passphrase): salt = get_random_bytes(32) iterations = 100000 key = pbkdf2_sha256(passphrase.encode('utf-8'), salt, iterations, 32) hash = HMAC(key, data, sha256).digest() cdata = AES(key).encrypt(data) d = { 'version': 1, 'salt': salt, 'iterations': iterations, 'algorithm': 'sha256', 'hash': hash, 'data': cdata, } return msgpack.packb(d) def save(self, path, passphrase): key = { 'version': 1, 'repository_id': self.repository_id, 'enc_key': self.enc_key, 'enc_hmac_key': self.enc_hmac_key, 'id_key': self.id_key, 'chunk_seed': self.chunk_seed, } data = self.encrypt_key_file(msgpack.packb(key), passphrase) with open(path, 'w') as fd: fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii'))) fd.write('\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))) self.path = path def change_passphrase(self): passphrase, passphrase2 = 1, 2 while passphrase != passphrase2: passphrase = getpass('New passphrase: ') passphrase2 = getpass('Enter same passphrase again: ') if passphrase != passphrase2: print('Passphrases do not match') self.save(self.path, passphrase) print('Key file "%s" updated' % self.path) @classmethod def create(cls, repository, args): filename = args.repository.to_key_filename() path = filename i = 1 while os.path.exists(path): i += 1 path = filename + '.%d' % i passphrase = os.environ.get('ATTIC_PASSPHRASE') if passphrase is not None: passphrase2 = passphrase else: passphrase, passphrase2 = 1, 2 while passphrase != passphrase2: passphrase = getpass('Enter passphrase (empty for no passphrase):') passphrase2 = getpass('Enter same passphrase again: ') if passphrase != passphrase2: print('Passphrases do not match') key = cls() key.repository_id = repository.id key.init_from_random_data(get_random_bytes(100)) key.init_ciphers() key.save(path, passphrase) print('Key file "%s" created.' % key.path) print('Keep this file safe. Your data will be inaccessible without it.') return key Attic-0.10/attic/lrucache.py0000644000175000017500000000221712267034246016220 0ustar jonasjonas00000000000000class LRUCache(dict): def __init__(self, capacity): super(LRUCache, self).__init__() self._lru = [] self._capacity = capacity def __setitem__(self, key, value): try: self._lru.remove(key) except ValueError: pass self._lru.append(key) while len(self._lru) > self._capacity: del self[self._lru[0]] return super(LRUCache, self).__setitem__(key, value) def __getitem__(self, key): try: self._lru.remove(key) self._lru.append(key) except ValueError: pass return super(LRUCache, self).__getitem__(key) def __delitem__(self, key): try: self._lru.remove(key) except ValueError: pass return super(LRUCache, self).__delitem__(key) def pop(self, key, default=None): try: self._lru.remove(key) except ValueError: pass return super(LRUCache, self).pop(key, default) def _not_implemented(self, *args, **kw): raise NotImplementedError popitem = setdefault = update = _not_implemented Attic-0.10/attic/remote.py0000644000175000017500000001677612272537704015747 0ustar jonasjonas00000000000000import fcntl import msgpack import os import select from subprocess import Popen, PIPE import sys from .helpers import Error from .repository import Repository BUFSIZE = 10 * 1024 * 1024 class ConnectionClosed(Error): """Connection closed by remote host""" class RepositoryServer(object): def __init__(self): self.repository = None def serve(self): # Make stdin non-blocking fl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL) fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, fl | os.O_NONBLOCK) # Make stdout blocking fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL) fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl & ~os.O_NONBLOCK) unpacker = msgpack.Unpacker(use_list=False) while True: r, w, es = select.select([sys.stdin], [], [], 10) if r: data = os.read(sys.stdin.fileno(), BUFSIZE) if not data: return unpacker.feed(data) for type, msgid, method, args in unpacker: method = method.decode('ascii') try: try: f = getattr(self, method) except AttributeError: f = getattr(self.repository, method) res = f(*args) except Exception as e: sys.stdout.buffer.write(msgpack.packb((1, msgid, e.__class__.__name__, e.args))) else: sys.stdout.buffer.write(msgpack.packb((1, msgid, None, res))) sys.stdout.flush() if es: return def negotiate(self, versions): return 1 def open(self, path, create=False): path = os.fsdecode(path) if path.startswith('/~'): path = path[1:] self.repository = Repository(os.path.expanduser(path), create) return self.repository.id class RemoteRepository(object): class RPCError(Exception): def __init__(self, name): self.name = name def __init__(self, location, create=False): self.location = location self.preload_ids = [] self.msgid = 0 self.to_send = b'' self.cache = {} self.ignore_responses = set() self.responses = {} self.unpacker = msgpack.Unpacker(use_list=False) self.p = None if location.host == '__testsuite__': args = [sys.executable, '-m', 'attic.archiver', 'serve'] else: args = ['ssh'] if location.port: args += ['-p', str(location.port)] if location.user: args.append('%s@%s' % (location.user, location.host)) else: args.append('%s' % location.host) args += ['attic', 'serve'] self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK) self.r_fds = [self.stdout_fd] self.x_fds = [self.stdin_fd, self.stdout_fd] version = self.call('negotiate', 1) if version != 1: raise Exception('Server insisted on using unsupported protocol version %d' % version) self.id = self.call('open', location.path, create) def __del__(self): self.close() def call(self, cmd, *args, **kw): for resp in self.call_many(cmd, [args], **kw): return resp def call_many(self, cmd, calls, wait=True, is_preloaded=False): if not calls: return def fetch_from_cache(args): msgid = self.cache[args].pop(0) if not self.cache[args]: del self.cache[args] return msgid calls = list(calls) waiting_for = [] w_fds = [self.stdin_fd] while wait or calls: while waiting_for: try: error, res = self.responses.pop(waiting_for[0]) waiting_for.pop(0) if error: if error == b'DoesNotExist': raise Repository.DoesNotExist(self.location.orig) elif error == b'AlreadyExists': raise Repository.AlreadyExists(self.location.orig) raise self.RPCError(error) else: yield res if not waiting_for and not calls: return except KeyError: break r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1) if x: raise Exception('FD exception occured') if r: data = os.read(self.stdout_fd, BUFSIZE) if not data: raise ConnectionClosed() self.unpacker.feed(data) for type, msgid, error, res in self.unpacker: if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) else: self.responses[msgid] = error, res if w: while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < 100: if calls: if is_preloaded: if calls[0] in self.cache: waiting_for.append(fetch_from_cache(calls.pop(0))) else: args = calls.pop(0) if cmd == 'get' and args in self.cache: waiting_for.append(fetch_from_cache(args)) else: self.msgid += 1 waiting_for.append(self.msgid) self.to_send = msgpack.packb((1, self.msgid, cmd, args)) if not self.to_send and self.preload_ids: args = (self.preload_ids.pop(0),) self.msgid += 1 self.cache.setdefault(args, []).append(self.msgid) self.to_send = msgpack.packb((1, self.msgid, cmd, args)) if self.to_send: self.to_send = self.to_send[os.write(self.stdin_fd, self.to_send):] if not self.to_send and not (calls or self.preload_ids): w_fds = [] self.ignore_responses |= set(waiting_for) def commit(self, *args): return self.call('commit') def rollback(self, *args): return self.call('rollback') def get(self, id_): for resp in self.get_many([id_]): return resp def get_many(self, ids, is_preloaded=False): for resp in self.call_many('get', [(id_,) for id_ in ids], is_preloaded=is_preloaded): yield resp def put(self, id_, data, wait=True): return self.call('put', id_, data, wait=wait) def delete(self, id_, wait=True): return self.call('delete', id_, wait=wait) def close(self): if self.p: self.p.stdin.close() self.p.stdout.close() self.p.wait() self.p = None def preload(self, ids): self.preload_ids += ids Attic-0.10/attic/repository.py0000644000175000017500000003721712270021266016651 0ustar jonasjonas00000000000000from configparser import RawConfigParser from binascii import hexlify import errno import os import re import shutil import struct from zlib import crc32 from .hashindex import NSIndex from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, UpgradableLock from .lrucache import LRUCache MAX_OBJECT_SIZE = 20 * 1024 * 1024 MAGIC = b'ATTICSEG' TAG_PUT = 0 TAG_DELETE = 1 TAG_COMMIT = 2 class Repository(object): """Filesystem based transactional key value store On disk layout: dir/README dir/config dir/data// dir/index.X dir/hints.X """ DEFAULT_MAX_SEGMENT_SIZE = 5 * 1024 * 1024 DEFAULT_SEGMENTS_PER_DIR = 10000 class DoesNotExist(Error): """Repository {} does not exist""" class AlreadyExists(Error): """Repository {} already exists""" class InvalidRepository(Error): """{} is not a valid repository""" def __init__(self, path, create=False): self.path = path self.io = None self.lock = None if create: self.create(path) self.open(path) def __del__(self): self.close() def create(self, path): """Create a new empty repository at `path` """ if os.path.exists(path) and (not os.path.isdir(path) or os.listdir(path)): raise self.AlreadyExists(path) if not os.path.exists(path): os.mkdir(path) with open(os.path.join(path, 'README'), 'w') as fd: fd.write('This is an Attic repository\n') os.mkdir(os.path.join(path, 'data')) config = RawConfigParser() config.add_section('repository') config.set('repository', 'version', '1') config.set('repository', 'segments_per_dir', self.DEFAULT_SEGMENTS_PER_DIR) config.set('repository', 'max_segment_size', self.DEFAULT_MAX_SEGMENT_SIZE) config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii')) with open(os.path.join(path, 'config'), 'w') as fd: config.write(fd) def open(self, path): self.head = None self.path = path if not os.path.isdir(path): raise self.DoesNotExist(path) self.config = RawConfigParser() self.config.read(os.path.join(self.path, 'config')) if not 'repository' in self.config.sections() or self.config.getint('repository', 'version') != 1: raise self.InvalidRepository(path) self.lock = UpgradableLock(os.path.join(path, 'config')) self.max_segment_size = self.config.getint('repository', 'max_segment_size') self.segments_per_dir = self.config.getint('repository', 'segments_per_dir') self.id = unhexlify(self.config.get('repository', 'id').strip()) self.rollback() def close(self): if self.lock: self.rollback() self.lock.release() self.lock = None def commit(self, rollback=True): """Commit transaction """ self.io.write_commit() self.compact_segments() self.write_index() self.rollback() def _available_indices(self, reverse=False): names = [int(name[6:]) for name in os.listdir(self.path) if re.match('index\.\d+', name)] names.sort(reverse=reverse) return names def open_index(self, head, read_only=False): if head is None: self.index = NSIndex.create(os.path.join(self.path, 'index.tmp').encode('utf-8')) self.segments = {} self.compact = set() else: if read_only: self.index = NSIndex((os.path.join(self.path, 'index.%d') % head).encode('utf-8'), readonly=True) else: shutil.copy(os.path.join(self.path, 'index.%d' % head), os.path.join(self.path, 'index.tmp')) self.index = NSIndex(os.path.join(self.path, 'index.tmp').encode('utf-8')) hints = read_msgpack(os.path.join(self.path, 'hints.%d' % head)) if hints[b'version'] != 1: raise ValueError('Unknown hints file version: %d' % hints['version']) self.segments = hints[b'segments'] self.compact = set(hints[b'compact']) def write_index(self): hints = {b'version': 1, b'segments': self.segments, b'compact': list(self.compact)} write_msgpack(os.path.join(self.path, 'hints.%d' % self.io.head), hints) self.index.flush() os.rename(os.path.join(self.path, 'index.tmp'), os.path.join(self.path, 'index.%d' % self.io.head)) # Remove old indices current = '.%d' % self.io.head for name in os.listdir(self.path): if not name.startswith('index.') and not name.startswith('hints.'): continue if name.endswith(current): continue os.unlink(os.path.join(self.path, name)) def compact_segments(self): """Compact sparse segments by copying data into new segments """ if not self.compact: return def lookup(tag, key): return tag == TAG_PUT and self.index.get(key, (-1, -1))[0] == segment segments = self.segments for segment in sorted(self.compact): if segments[segment] > 0: for tag, key, data in self.io.iter_objects(segment, lookup, include_data=True): new_segment, offset = self.io.write_put(key, data) self.index[key] = new_segment, offset segments.setdefault(new_segment, 0) segments[new_segment] += 1 segments[segment] -= 1 assert segments[segment] == 0 self.io.write_commit() for segment in self.compact: assert self.segments.pop(segment) == 0 self.io.delete_segment(segment) self.compact = set() def recover(self, path): """Recover missing index by replaying logs""" start = None available = self._available_indices() if available: start = available[-1] self.open_index(start) for segment, filename in self.io._segment_names(): if start is not None and segment <= start: continue self.segments[segment] = 0 for tag, key, offset in self.io.iter_objects(segment): if tag == TAG_PUT: try: s, _ = self.index[key] self.compact.add(s) self.segments[s] -= 1 except KeyError: pass self.index[key] = segment, offset self.segments[segment] += 1 elif tag == TAG_DELETE: try: s, _ = self.index.pop(key) self.segments[s] -= 1 self.compact.add(s) self.compact.add(segment) except KeyError: pass if self.segments[segment] == 0: self.compact.add(segment) if self.io.head is not None: self.write_index() def rollback(self): """ """ self._active_txn = False if self.io: self.io.close() self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir) if self.io.head is not None and not os.path.exists(os.path.join(self.path, 'index.%d' % self.io.head)): self.lock.upgrade() self.recover(self.path) self.open_index(self.io.head, read_only=True) def _len(self): return len(self.index) def get(self, id): try: segment, offset = self.index[id] return self.io.read(segment, offset, id) except KeyError: raise self.DoesNotExist(self.path) def get_many(self, ids, is_preloaded=False): for id_ in ids: yield self.get(id_) def put(self, id, data, wait=True): if not self._active_txn: self._active_txn = True self.lock.upgrade() self.open_index(self.io.head) try: segment, _ = self.index[id] self.segments[segment] -= 1 self.compact.add(segment) segment = self.io.write_delete(id) self.segments.setdefault(segment, 0) self.compact.add(segment) except KeyError: pass segment, offset = self.io.write_put(id, data) self.segments.setdefault(segment, 0) self.segments[segment] += 1 self.index[id] = segment, offset def delete(self, id, wait=True): if not self._active_txn: self._active_txn = True self.lock.upgrade() self.open_index(self.io.head) try: segment, offset = self.index.pop(id) self.segments[segment] -= 1 self.compact.add(segment) segment = self.io.write_delete(id) self.compact.add(segment) self.segments.setdefault(segment, 0) except KeyError: raise self.DoesNotExist(self.path) def add_callback(self, cb, data): cb(None, None, data) def preload(self, ids): """Preload objects (only applies to remote repositories """ class LoggedIO(object): header_fmt = struct.Struct(' self.limit: self.close_segment() if not self._write_fd: if self.segment % self.segments_per_dir == 0: dirname = os.path.join(self.path, 'data', str(self.segment // self.segments_per_dir)) if not os.path.exists(dirname): os.mkdir(dirname) self._write_fd = open(self.segment_filename(self.segment), 'ab') self._write_fd.write(MAGIC) self.offset = 8 return self._write_fd def get_fd(self, segment): try: return self.fds[segment] except KeyError: fd = open(self.segment_filename(segment), 'rb') self.fds[segment] = fd return fd def delete_segment(self, segment): try: os.unlink(self.segment_filename(segment)) except OSError: pass def iter_objects(self, segment, lookup=None, include_data=False): fd = self.get_fd(segment) fd.seek(0) if fd.read(8) != MAGIC: raise IntegrityError('Invalid segment header') offset = 8 header = fd.read(self.header_fmt.size) while header: crc, size, tag = self.header_fmt.unpack(header) if size > MAX_OBJECT_SIZE: raise IntegrityError('Invalid segment object size') rest = fd.read(size - self.header_fmt.size) if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc: raise IntegrityError('Segment checksum mismatch') if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT): raise IntegrityError('Invalid segment entry header') key = None if tag in (TAG_PUT, TAG_DELETE): key = rest[:32] if not lookup or lookup(tag, key): if include_data: yield tag, key, rest[32:] else: yield tag, key, offset offset += size header = fd.read(self.header_fmt.size) def read(self, segment, offset, id): if segment == self.segment: self._write_fd.flush() fd = self.get_fd(segment) fd.seek(offset) header = fd.read(self.put_header_fmt.size) crc, size, tag, key = self.put_header_fmt.unpack(header) if size > MAX_OBJECT_SIZE: raise IntegrityError('Invalid segment object size') data = fd.read(size - self.put_header_fmt.size) if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc: raise IntegrityError('Segment checksum mismatch') if tag != TAG_PUT or id != key: raise IntegrityError('Invalid segment entry header') return data def write_put(self, id, data): size = len(data) + self.put_header_fmt.size fd = self.get_write_fd() offset = self.offset header = self.header_no_crc_fmt.pack(size, TAG_PUT) crc = self.crc_fmt.pack(crc32(data, crc32(id, crc32(header))) & 0xffffffff) fd.write(b''.join((crc, header, id, data))) self.offset += size return self.segment, offset def write_delete(self, id): fd = self.get_write_fd() header = self.header_no_crc_fmt.pack(self.put_header_fmt.size, TAG_DELETE) crc = self.crc_fmt.pack(crc32(id, crc32(header)) & 0xffffffff) fd.write(b''.join((crc, header, id))) self.offset += self.put_header_fmt.size return self.segment def write_commit(self): fd = self.get_write_fd(no_new=True) header = self.header_no_crc_fmt.pack(self.header_fmt.size, TAG_COMMIT) crc = self.crc_fmt.pack(crc32(header) & 0xffffffff) fd.write(b''.join((crc, header))) self.head = self.segment self.close_segment() def close_segment(self): if self._write_fd: self.segment += 1 self.offset = 0 os.fsync(self._write_fd) self._write_fd.close() self._write_fd = None Attic-0.10/attic/xattr.py0000644000175000017500000002214312210717760015571 0ustar jonasjonas00000000000000"""A basic extended attributes (xattr) implementation for Linux and MacOS X """ import errno import os import sys import tempfile from ctypes import CDLL, create_string_buffer, c_ssize_t, c_size_t, c_char_p, c_int, c_uint32, get_errno from ctypes.util import find_library def is_enabled(): """Determine if xattr is enabled on the filesystem """ with tempfile.NamedTemporaryFile() as fd: try: setxattr(fd.fileno(), 'user.name', b'value') except OSError: return False return getxattr(fd.fileno(), 'user.name') == b'value' def get_all(path, follow_symlinks=True): try: return dict((name, getxattr(path, name, follow_symlinks=follow_symlinks)) for name in listxattr(path, follow_symlinks=follow_symlinks)) except OSError as e: if e.errno in (errno.ENOTSUP, errno.EPERM): return {} libc = CDLL(find_library('c'), use_errno=True) def _check(rv, path=None): if rv < 0: raise OSError(get_errno(), path) return rv if sys.platform.startswith('linux'): libc.llistxattr.argtypes = (c_char_p, c_char_p, c_size_t) libc.llistxattr.restype = c_ssize_t libc.flistxattr.argtypes = (c_int, c_char_p, c_size_t) libc.flistxattr.restype = c_ssize_t libc.lsetxattr.argtypes = (c_char_p, c_char_p, c_char_p, c_size_t, c_int) libc.lsetxattr.restype = c_int libc.fsetxattr.argtypes = (c_int, c_char_p, c_char_p, c_size_t, c_int) libc.fsetxattr.restype = c_int libc.lgetxattr.argtypes = (c_char_p, c_char_p, c_char_p, c_size_t) libc.lgetxattr.restype = c_ssize_t libc.fgetxattr.argtypes = (c_int, c_char_p, c_char_p, c_size_t) libc.fgetxattr.restype = c_ssize_t def listxattr(path, *, follow_symlinks=True): if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.flistxattr elif follow_symlinks: func = libc.listxattr else: func = libc.llistxattr n = _check(func(path, None, 0), path) if n == 0: return [] namebuf = create_string_buffer(n) n2 = _check(func(path, namebuf, n), path) if n2 != n: raise Exception('listxattr failed') return [os.fsdecode(name) for name in namebuf.raw.split(b'\0')[:-1]] def getxattr(path, name, *, follow_symlinks=True): name = os.fsencode(name) if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.fgetxattr elif follow_symlinks: func = libc.getxattr else: func = libc.lgetxattr n = _check(func(path, name, None, 0)) if n == 0: return valuebuf = create_string_buffer(n) n2 = _check(func(path, name, valuebuf, n), path) if n2 != n: raise Exception('getxattr failed') return valuebuf.raw def setxattr(path, name, value, *, follow_symlinks=True): name = os.fsencode(name) value = os.fsencode(value) if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.fsetxattr elif follow_symlinks: func = libc.setxattr else: func = libc.lsetxattr _check(func(path, name, value, len(value), 0), path) elif sys.platform == 'darwin': libc.listxattr.argtypes = (c_char_p, c_char_p, c_size_t, c_int) libc.listxattr.restype = c_ssize_t libc.flistxattr.argtypes = (c_int, c_char_p, c_size_t) libc.flistxattr.restype = c_ssize_t libc.setxattr.argtypes = (c_char_p, c_char_p, c_char_p, c_size_t, c_uint32, c_int) libc.setxattr.restype = c_int libc.fsetxattr.argtypes = (c_int, c_char_p, c_char_p, c_size_t, c_uint32, c_int) libc.fsetxattr.restype = c_int libc.getxattr.argtypes = (c_char_p, c_char_p, c_char_p, c_size_t, c_uint32, c_int) libc.getxattr.restype = c_ssize_t libc.fgetxattr.argtypes = (c_int, c_char_p, c_char_p, c_size_t, c_uint32, c_int) libc.fgetxattr.restype = c_ssize_t XATTR_NOFOLLOW = 0x0001 def listxattr(path, *, follow_symlinks=True): func = libc.listxattr flags = 0 if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.flistxattr elif not follow_symlinks: flags = XATTR_NOFOLLOW n = _check(func(path, None, 0, flags), path) if n == 0: return [] namebuf = create_string_buffer(n) n2 = _check(func(path, namebuf, n, flags), path) if n2 != n: raise Exception('listxattr failed') return [os.fsdecode(name) for name in namebuf.raw.split(b'\0')[:-1]] def getxattr(path, name, *, follow_symlinks=True): name = os.fsencode(name) func = libc.getxattr flags = 0 if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.fgetxattr elif not follow_symlinks: flags = XATTR_NOFOLLOW n = _check(func(path, name, None, 0, 0, flags)) if n == 0: return valuebuf = create_string_buffer(n) n2 = _check(func(path, name, valuebuf, n, 0, flags), path) if n2 != n: raise Exception('getxattr failed') return valuebuf.raw def setxattr(path, name, value, *, follow_symlinks=True): name = os.fsencode(name) value = os.fsencode(value) func = libc.setxattr flags = 0 if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.fsetxattr elif not follow_symlinks: flags = XATTR_NOFOLLOW _check(func(path, name, value, len(value), 0, flags), path) elif sys.platform.startswith('freebsd'): EXTATTR_NAMESPACE_USER = 0x0001 libc.extattr_list_fd.argtypes = (c_int, c_int, c_char_p, c_size_t) libc.extattr_list_fd.restype = c_ssize_t libc.extattr_list_link.argtypes = (c_char_p, c_int, c_char_p, c_size_t) libc.extattr_list_link.restype = c_ssize_t libc.extattr_list_file.argtypes = (c_char_p, c_int, c_char_p, c_size_t) libc.extattr_list_file.restype = c_ssize_t libc.extattr_get_fd.argtypes = (c_int, c_int, c_char_p, c_char_p, c_size_t) libc.extattr_get_fd.restype = c_ssize_t libc.extattr_get_link.argtypes = (c_char_p, c_int, c_char_p, c_char_p, c_size_t) libc.extattr_get_link.restype = c_ssize_t libc.extattr_get_file.argtypes = (c_char_p, c_int, c_char_p, c_char_p, c_size_t) libc.extattr_get_file.restype = c_ssize_t libc.extattr_set_fd.argtypes = (c_int, c_int, c_char_p, c_char_p, c_size_t) libc.extattr_set_fd.restype = c_int libc.extattr_set_link.argtypes = (c_char_p, c_int, c_char_p, c_char_p, c_size_t) libc.extattr_set_link.restype = c_int libc.extattr_set_file.argtypes = (c_char_p, c_int, c_char_p, c_char_p, c_size_t) libc.extattr_set_file.restype = c_int def listxattr(path, *, follow_symlinks=True): ns = EXTATTR_NAMESPACE_USER if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.extattr_list_fd elif follow_symlinks: func = libc.extattr_list_file else: func = libc.extattr_list_link n = _check(func(path, ns, None, 0), path) if n == 0: return [] namebuf = create_string_buffer(n) n2 = _check(func(path, ns, namebuf, n), path) if n2 != n: raise Exception('listxattr failed') names = [] mv = memoryview(namebuf.raw) while mv: length = mv[0] # Python < 3.3 returns bytes instead of int if isinstance(length, bytes): length = ord(length) names.append(os.fsdecode(bytes(mv[1:1+length]))) mv = mv[1+length:] return names def getxattr(path, name, *, follow_symlinks=True): name = os.fsencode(name) if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.extattr_get_fd elif follow_symlinks: func = libc.extattr_get_file else: func = libc.extattr_get_link n = _check(func(path, EXTATTR_NAMESPACE_USER, name, None, 0)) if n == 0: return valuebuf = create_string_buffer(n) n2 = _check(func(path, EXTATTR_NAMESPACE_USER, name, valuebuf, n), path) if n2 != n: raise Exception('getxattr failed') return valuebuf.raw def setxattr(path, name, value, *, follow_symlinks=True): name = os.fsencode(name) value = os.fsencode(value) if isinstance(path, str): path = os.fsencode(path) if isinstance(path, int): func = libc.extattr_set_fd elif follow_symlinks: func = libc.extattr_set_file else: func = libc.extattr_set_link _check(func(path, EXTATTR_NAMESPACE_USER, name, value, len(value)), path) else: raise Exception('Unsupported platform: %s' % sys.platform) Attic-0.10/attic/chunker.c0000644000175000017500000032551412272541113015663 0ustar jonasjonas00000000000000/* Generated by Cython 0.19.1 on Thu Jan 30 22:21:14 2014 */ #define PY_SSIZE_T_CLEAN #ifndef CYTHON_USE_PYLONG_INTERNALS #ifdef PYLONG_BITS_IN_DIGIT #define CYTHON_USE_PYLONG_INTERNALS 0 #else #include "pyconfig.h" #ifdef PYLONG_BITS_IN_DIGIT #define CYTHON_USE_PYLONG_INTERNALS 1 #else #define CYTHON_USE_PYLONG_INTERNALS 0 #endif #endif #endif #include "Python.h" #ifndef Py_PYTHON_H #error Python headers needed to compile C extensions, please install development version of Python. #elif PY_VERSION_HEX < 0x02040000 #error Cython requires Python 2.4+. #else #include /* For offsetof */ #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif #if !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif #ifndef __cdecl #define __cdecl #endif #ifndef __fastcall #define __fastcall #endif #endif #ifndef DL_IMPORT #define DL_IMPORT(t) t #endif #ifndef DL_EXPORT #define DL_EXPORT(t) t #endif #ifndef PY_LONG_LONG #define PY_LONG_LONG LONG_LONG #endif #ifndef Py_HUGE_VAL #define Py_HUGE_VAL HUGE_VAL #endif #ifdef PYPY_VERSION #define CYTHON_COMPILING_IN_PYPY 1 #define CYTHON_COMPILING_IN_CPYTHON 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_CPYTHON 1 #endif #if PY_VERSION_HEX < 0x02050000 typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX #define PY_SSIZE_T_MIN INT_MIN #define PY_FORMAT_SIZE_T "" #define CYTHON_FORMAT_SSIZE_T "" #define PyInt_FromSsize_t(z) PyInt_FromLong(z) #define PyInt_AsSsize_t(o) __Pyx_PyInt_AsInt(o) #define PyNumber_Index(o) ((PyNumber_Check(o) && !PyFloat_Check(o)) ? PyNumber_Int(o) : \ (PyErr_Format(PyExc_TypeError, \ "expected index value, got %.200s", Py_TYPE(o)->tp_name), \ (PyObject*)0)) #define __Pyx_PyIndex_Check(o) (PyNumber_Check(o) && !PyFloat_Check(o) && \ !PyComplex_Check(o)) #define PyIndex_Check __Pyx_PyIndex_Check #define PyErr_WarnEx(category, message, stacklevel) PyErr_Warn(category, message) #define __PYX_BUILD_PY_SSIZE_T "i" #else #define __PYX_BUILD_PY_SSIZE_T "n" #define CYTHON_FORMAT_SSIZE_T "z" #define __Pyx_PyIndex_Check PyIndex_Check #endif #if PY_VERSION_HEX < 0x02060000 #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) #define PyVarObject_HEAD_INIT(type, size) \ PyObject_HEAD_INIT(type) size, #define PyType_Modified(t) typedef struct { void *buf; PyObject *obj; Py_ssize_t len; Py_ssize_t itemsize; int readonly; int ndim; char *format; Py_ssize_t *shape; Py_ssize_t *strides; Py_ssize_t *suboffsets; void *internal; } Py_buffer; #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 #define PyBUF_FORMAT 0x0004 #define PyBUF_ND 0x0008 #define PyBUF_STRIDES (0x0010 | PyBUF_ND) #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) #define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_FORMAT | PyBUF_WRITABLE) #define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_FORMAT | PyBUF_WRITABLE) typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); typedef void (*releasebufferproc)(PyObject *, Py_buffer *); #endif #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #endif #if PY_MAJOR_VERSION < 3 && PY_MINOR_VERSION < 6 #define PyUnicode_FromString(s) PyUnicode_Decode(s, strlen(s), "UTF-8", "strict") #endif #if PY_MAJOR_VERSION >= 3 #define Py_TPFLAGS_CHECKTYPES 0 #define Py_TPFLAGS_HAVE_INDEX 0 #endif #if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) #define Py_TPFLAGS_HAVE_NEWBUFFER 0 #endif #if PY_VERSION_HEX < 0x02060000 #define Py_TPFLAGS_HAVE_VERSION_TAG 0 #endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #else #define CYTHON_PEP393_ENABLED 0 #define __Pyx_PyUnicode_READY(op) (0) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) #define __Pyx_PyUnicode_READ(k, d, i) ((k=k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #endif #if PY_MAJOR_VERSION >= 3 #define PyBaseString_Type PyUnicode_Type #define PyStringObject PyUnicodeObject #define PyString_Type PyUnicode_Type #define PyString_Check PyUnicode_Check #define PyString_CheckExact PyUnicode_CheckExact #endif #if PY_VERSION_HEX < 0x02060000 #define PyBytesObject PyStringObject #define PyBytes_Type PyString_Type #define PyBytes_Check PyString_Check #define PyBytes_CheckExact PyString_CheckExact #define PyBytes_FromString PyString_FromString #define PyBytes_FromStringAndSize PyString_FromStringAndSize #define PyBytes_FromFormat PyString_FromFormat #define PyBytes_DecodeEscape PyString_DecodeEscape #define PyBytes_AsString PyString_AsString #define PyBytes_AsStringAndSize PyString_AsStringAndSize #define PyBytes_Size PyString_Size #define PyBytes_AS_STRING PyString_AS_STRING #define PyBytes_GET_SIZE PyString_GET_SIZE #define PyBytes_Repr PyString_Repr #define PyBytes_Concat PyString_Concat #define PyBytes_ConcatAndDel PyString_ConcatAndDel #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) #else #define __Pyx_PyBaseString_Check(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj) || \ PyString_Check(obj) || PyUnicode_Check(obj)) #define __Pyx_PyBaseString_CheckExact(obj) (Py_TYPE(obj) == &PyBaseString_Type) #endif #if PY_VERSION_HEX < 0x02060000 #define PySet_Check(obj) PyObject_TypeCheck(obj, &PySet_Type) #define PyFrozenSet_Check(obj) PyObject_TypeCheck(obj, &PyFrozenSet_Type) #endif #ifndef PySet_CheckExact #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject #define PyInt_Type PyLong_Type #define PyInt_Check(op) PyLong_Check(op) #define PyInt_CheckExact(op) PyLong_CheckExact(op) #define PyInt_FromString PyLong_FromString #define PyInt_FromUnicode PyLong_FromUnicode #define PyInt_FromLong PyLong_FromLong #define PyInt_FromSize_t PyLong_FromSize_t #define PyInt_FromSsize_t PyLong_FromSsize_t #define PyInt_AsLong PyLong_AsLong #define PyInt_AS_LONG PyLong_AS_LONG #define PyInt_AsSsize_t PyLong_AsSsize_t #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask #endif #if PY_MAJOR_VERSION >= 3 #define PyBoolObject PyLongObject #endif #if PY_VERSION_HEX < 0x03020000 typedef long Py_hash_t; #define __Pyx_PyInt_FromHash_t PyInt_FromLong #define __Pyx_PyInt_AsHash_t PyInt_AsLong #else #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t #endif #if (PY_MAJOR_VERSION < 3) || (PY_VERSION_HEX >= 0x03010300) #define __Pyx_PySequence_GetSlice(obj, a, b) PySequence_GetSlice(obj, a, b) #define __Pyx_PySequence_SetSlice(obj, a, b, value) PySequence_SetSlice(obj, a, b, value) #define __Pyx_PySequence_DelSlice(obj, a, b) PySequence_DelSlice(obj, a, b) #else #define __Pyx_PySequence_GetSlice(obj, a, b) (unlikely(!(obj)) ? \ (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), (PyObject*)0) : \ (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_GetSlice(obj, a, b)) : \ (PyErr_Format(PyExc_TypeError, "'%.200s' object is unsliceable", (obj)->ob_type->tp_name), (PyObject*)0))) #define __Pyx_PySequence_SetSlice(obj, a, b, value) (unlikely(!(obj)) ? \ (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \ (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_SetSlice(obj, a, b, value)) : \ (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice assignment", (obj)->ob_type->tp_name), -1))) #define __Pyx_PySequence_DelSlice(obj, a, b) (unlikely(!(obj)) ? \ (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \ (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_DelSlice(obj, a, b)) : \ (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice deletion", (obj)->ob_type->tp_name), -1))) #endif #if PY_MAJOR_VERSION >= 3 #define PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) #endif #if PY_VERSION_HEX < 0x02050000 #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) #else #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) #endif #if PY_VERSION_HEX < 0x02050000 #define __Pyx_NAMESTR(n) ((char *)(n)) #define __Pyx_DOCSTR(n) ((char *)(n)) #else #define __Pyx_NAMESTR(n) (n) #define __Pyx_DOCSTR(n) (n) #endif #ifndef CYTHON_INLINE #if defined(__GNUC__) #define CYTHON_INLINE __inline__ #elif defined(_MSC_VER) #define CYTHON_INLINE __inline #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define CYTHON_INLINE inline #else #define CYTHON_INLINE #endif #endif #ifndef CYTHON_RESTRICT #if defined(__GNUC__) #define CYTHON_RESTRICT __restrict__ #elif defined(_MSC_VER) #define CYTHON_RESTRICT __restrict #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define CYTHON_RESTRICT restrict #else #define CYTHON_RESTRICT #endif #endif #ifdef NAN #define __PYX_NAN() ((float) NAN) #else static CYTHON_INLINE float __PYX_NAN() { /* Initialize NaN. The sign is irrelevant, an exponent with all bits 1 and a nonzero mantissa means NaN. If the first bit in the mantissa is 1, it is a quiet NaN. */ float value; memset(&value, 0xFF, sizeof(value)); return value; } #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) #else #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif #ifndef __PYX_EXTERN_C #ifdef __cplusplus #define __PYX_EXTERN_C extern "C" #else #define __PYX_EXTERN_C extern #endif #endif #if defined(WIN32) || defined(MS_WINDOWS) #define _USE_MATH_DEFINES #endif #include #define __PYX_HAVE__attic__chunker #define __PYX_HAVE_API__attic__chunker #include "string.h" #include "stdlib.h" #include "_chunker.c" #ifdef _OPENMP #include #endif /* _OPENMP */ #ifdef PYREX_WITHOUT_ASSERTIONS #define CYTHON_WITHOUT_ASSERTIONS #endif #ifndef CYTHON_UNUSED # if defined(__GNUC__) # if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) # define CYTHON_UNUSED __attribute__ ((__unused__)) # else # define CYTHON_UNUSED # endif # elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) # define CYTHON_UNUSED __attribute__ ((__unused__)) # else # define CYTHON_UNUSED # endif #endif typedef struct {PyObject **p; char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/ #define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 #define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 #define __PYX_DEFAULT_STRING_ENCODING "" #define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString #define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); #define __Pyx_PyBytes_FromString PyBytes_FromString #define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char*); #if PY_MAJOR_VERSION < 3 #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize #else #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize #endif #define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromUString(s) __Pyx_PyObject_FromString((char*)s) #define __Pyx_PyBytes_FromUString(s) __Pyx_PyBytes_FromString((char*)s) #define __Pyx_PyStr_FromUString(s) __Pyx_PyStr_FromString((char*)s) #define __Pyx_PyUnicode_FromUString(s) __Pyx_PyUnicode_FromString((char*)s) #if PY_MAJOR_VERSION < 3 static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { const Py_UNICODE *u_end = u; while (*u_end++) ; return u_end - u - 1; } #else #define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen #endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode #define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None) #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); #if CYTHON_COMPILING_IN_CPYTHON #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) #else #define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) #endif #define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; static int __Pyx_init_sys_getdefaultencoding_params() { PyObject* sys = NULL; PyObject* default_encoding = NULL; PyObject* ascii_chars_u = NULL; PyObject* ascii_chars_b = NULL; sys = PyImport_ImportModule("sys"); if (sys == NULL) goto bad; default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); if (default_encoding == NULL) goto bad; if (strcmp(PyBytes_AsString(default_encoding), "ascii") == 0) { __Pyx_sys_getdefaultencoding_not_ascii = 0; } else { const char* default_encoding_c = PyBytes_AS_STRING(default_encoding); char ascii_chars[128]; int c; for (c = 0; c < 128; c++) { ascii_chars[c] = c; } __Pyx_sys_getdefaultencoding_not_ascii = 1; ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); if (ascii_chars_u == NULL) goto bad; ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); if (ascii_chars_b == NULL || strncmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { PyErr_Format( PyExc_ValueError, "This module compiled with c_string_encoding=ascii, but default encoding '%s' is not a superset of ascii.", default_encoding_c); goto bad; } } Py_XDECREF(sys); Py_XDECREF(default_encoding); Py_XDECREF(ascii_chars_u); Py_XDECREF(ascii_chars_b); return 0; bad: Py_XDECREF(sys); Py_XDECREF(default_encoding); Py_XDECREF(ascii_chars_u); Py_XDECREF(ascii_chars_b); return -1; } #endif #if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) #else #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) #if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT static char* __PYX_DEFAULT_STRING_ENCODING; static int __Pyx_init_sys_getdefaultencoding_params() { PyObject* sys = NULL; PyObject* default_encoding = NULL; char* default_encoding_c; sys = PyImport_ImportModule("sys"); if (sys == NULL) goto bad; default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); if (default_encoding == NULL) goto bad; default_encoding_c = PyBytes_AS_STRING(default_encoding); __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); Py_DECREF(sys); Py_DECREF(default_encoding); return 0; bad: Py_XDECREF(sys); Py_XDECREF(default_encoding); return -1; } #endif #endif #ifdef __GNUC__ /* Test for GCC > 2.95 */ #if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #else /* __GNUC__ > 2 ... */ #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ > 2 ... */ #else /* __GNUC__ */ #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ */ static PyObject *__pyx_m; static PyObject *__pyx_d; static PyObject *__pyx_b; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; static int __pyx_lineno; static int __pyx_clineno = 0; static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char *__pyx_f[] = { "chunker.pyx", }; /*--- Type declarations ---*/ struct __pyx_obj_5attic_7chunker_chunkify; /* "attic/chunker.pyx":17 * * * cdef class chunkify: # <<<<<<<<<<<<<< * cdef Chunker *chunker * */ struct __pyx_obj_5attic_7chunker_chunkify { PyObject_HEAD Chunker *chunker; }; #ifndef CYTHON_REFNANNY #define CYTHON_REFNANNY 0 #endif #if CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); void (*DECREF)(void*, PyObject*, int); void (*GOTREF)(void*, PyObject*, int); void (*GIVEREF)(void*, PyObject*, int); void* (*SetupContext)(const char*, int, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); /*proto*/ #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; #ifdef WITH_THREAD #define __Pyx_RefNannySetupContext(name, acquire_gil) \ if (acquire_gil) { \ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ PyGILState_Release(__pyx_gilstate_save); \ } else { \ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil) \ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) #endif #define __Pyx_RefNannyFinishContext() \ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) #define __Pyx_GOTREF(r) #define __Pyx_GIVEREF(r) #define __Pyx_XINCREF(r) Py_XINCREF(r) #define __Pyx_XDECREF(r) Py_XDECREF(r) #define __Pyx_XGOTREF(r) #define __Pyx_XGIVEREF(r) #endif /* CYTHON_REFNANNY */ #define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) #define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); /*proto*/ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ const char* function_name); /*proto*/ #include static CYTHON_INLINE uint32_t __Pyx_PyInt_from_py_uint32_t(PyObject *); static CYTHON_INLINE PyObject *__Pyx_PyInt_to_py_uint32_t(uint32_t); static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject *); static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject *); static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject *); static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject *); static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject *); static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); static int __Pyx_check_binary_version(void); typedef struct { int code_line; PyCodeObject* code_object; } __Pyx_CodeObjectCacheEntry; struct __Pyx_CodeObjectCache { int count; int max_count; __Pyx_CodeObjectCacheEntry* entries; }; static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename); /*proto*/ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Module declarations from 'libc.string' */ /* Module declarations from 'libc.stdlib' */ /* Module declarations from 'attic.chunker' */ static PyTypeObject *__pyx_ptype_5attic_7chunker_chunkify = 0; #define __Pyx_MODULE_NAME "attic.chunker" int __pyx_module_is_main_attic__chunker = 0; /* Implementation of 'attic.chunker' */ static int __pyx_pf_5attic_7chunker_8chunkify___cinit__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self, PyObject *__pyx_v_fd, PyObject *__pyx_v_window_size, PyObject *__pyx_v_chunk_mask, PyObject *__pyx_v_min_size, PyObject *__pyx_v_seed); /* proto */ static void __pyx_pf_5attic_7chunker_8chunkify_2__dealloc__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_7chunker_8chunkify_4__iter__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_7chunker_8chunkify_6__next__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_7chunker_buzhash(CYTHON_UNUSED PyObject *__pyx_self, unsigned char *__pyx_v_data, unsigned long __pyx_v_seed); /* proto */ static PyObject *__pyx_pf_5attic_7chunker_2buzhash_update(CYTHON_UNUSED PyObject *__pyx_self, uint32_t __pyx_v_sum, unsigned char __pyx_v_remove, unsigned char __pyx_v_add, size_t __pyx_v_len, unsigned long __pyx_v_seed); /* proto */ static PyObject *__pyx_tp_new_5attic_7chunker_chunkify(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static char __pyx_k_3[] = "/home/jonas/src/attic/attic/chunker.pyx"; static char __pyx_k_4[] = "attic.chunker"; static char __pyx_k__fd[] = "fd"; static char __pyx_k__add[] = "add"; static char __pyx_k__len[] = "len"; static char __pyx_k__sum[] = "sum"; static char __pyx_k__data[] = "data"; static char __pyx_k__seed[] = "seed"; static char __pyx_k__table[] = "table"; static char __pyx_k__remove[] = "remove"; static char __pyx_k__buzhash[] = "buzhash"; static char __pyx_k____main__[] = "__main__"; static char __pyx_k____test__[] = "__test__"; static char __pyx_k__min_size[] = "min_size"; static char __pyx_k__chunk_mask[] = "chunk_mask"; static char __pyx_k__window_size[] = "window_size"; static char __pyx_k__buzhash_update[] = "buzhash_update"; static PyObject *__pyx_kp_s_3; static PyObject *__pyx_n_s_4; static PyObject *__pyx_n_s____main__; static PyObject *__pyx_n_s____test__; static PyObject *__pyx_n_s__add; static PyObject *__pyx_n_s__buzhash; static PyObject *__pyx_n_s__buzhash_update; static PyObject *__pyx_n_s__chunk_mask; static PyObject *__pyx_n_s__data; static PyObject *__pyx_n_s__fd; static PyObject *__pyx_n_s__len; static PyObject *__pyx_n_s__min_size; static PyObject *__pyx_n_s__remove; static PyObject *__pyx_n_s__seed; static PyObject *__pyx_n_s__sum; static PyObject *__pyx_n_s__table; static PyObject *__pyx_n_s__window_size; static PyObject *__pyx_int_4294967295; static PyObject *__pyx_k_tuple_1; static PyObject *__pyx_k_tuple_5; static PyObject *__pyx_k_codeobj_2; static PyObject *__pyx_k_codeobj_6; /* Python wrapper */ static int __pyx_pw_5attic_7chunker_8chunkify_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5attic_7chunker_8chunkify_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_fd = 0; PyObject *__pyx_v_window_size = 0; PyObject *__pyx_v_chunk_mask = 0; PyObject *__pyx_v_min_size = 0; PyObject *__pyx_v_seed = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__fd,&__pyx_n_s__window_size,&__pyx_n_s__chunk_mask,&__pyx_n_s__min_size,&__pyx_n_s__seed,0}; PyObject* values[5] = {0,0,0,0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__fd)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__window_size)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 5, 5, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__chunk_mask)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 5, 5, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__min_size)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 5, 5, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 4: if (likely((values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__seed)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 5, 5, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 5) { goto __pyx_L5_argtuple_error; } else { values[0] = PyTuple_GET_ITEM(__pyx_args, 0); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[3] = PyTuple_GET_ITEM(__pyx_args, 3); values[4] = PyTuple_GET_ITEM(__pyx_args, 4); } __pyx_v_fd = values[0]; __pyx_v_window_size = values[1]; __pyx_v_chunk_mask = values[2]; __pyx_v_min_size = values[3]; __pyx_v_seed = values[4]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 5, 5, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.chunker.chunkify.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_7chunker_8chunkify___cinit__(((struct __pyx_obj_5attic_7chunker_chunkify *)__pyx_v_self), __pyx_v_fd, __pyx_v_window_size, __pyx_v_chunk_mask, __pyx_v_min_size, __pyx_v_seed); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/chunker.pyx":20 * cdef Chunker *chunker * * def __cinit__(self, fd, window_size, chunk_mask, min_size, seed): # <<<<<<<<<<<<<< * self.chunker = chunker_init(fd, window_size, chunk_mask, min_size, seed & 0xffffffff) * */ static int __pyx_pf_5attic_7chunker_8chunkify___cinit__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self, PyObject *__pyx_v_fd, PyObject *__pyx_v_window_size, PyObject *__pyx_v_chunk_mask, PyObject *__pyx_v_min_size, PyObject *__pyx_v_seed) { int __pyx_r; __Pyx_RefNannyDeclarations int __pyx_t_1; int __pyx_t_2; int __pyx_t_3; PyObject *__pyx_t_4 = NULL; uint32_t __pyx_t_5; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); /* "attic/chunker.pyx":21 * * def __cinit__(self, fd, window_size, chunk_mask, min_size, seed): * self.chunker = chunker_init(fd, window_size, chunk_mask, min_size, seed & 0xffffffff) # <<<<<<<<<<<<<< * * def __dealloc__(self): */ __pyx_t_1 = __Pyx_PyInt_AsInt(__pyx_v_window_size); if (unlikely((__pyx_t_1 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_2 = __Pyx_PyInt_AsInt(__pyx_v_chunk_mask); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_3 = __Pyx_PyInt_AsInt(__pyx_v_min_size); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_4 = PyNumber_And(__pyx_v_seed, __pyx_int_4294967295); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __Pyx_PyInt_from_py_uint32_t(__pyx_t_4); if (unlikely((__pyx_t_5 == (uint32_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_v_self->chunker = chunker_init(__pyx_v_fd, __pyx_t_1, __pyx_t_2, __pyx_t_3, __pyx_t_5); __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("attic.chunker.chunkify.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static void __pyx_pw_5attic_7chunker_8chunkify_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ static void __pyx_pw_5attic_7chunker_8chunkify_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); __pyx_pf_5attic_7chunker_8chunkify_2__dealloc__(((struct __pyx_obj_5attic_7chunker_chunkify *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); } /* "attic/chunker.pyx":23 * self.chunker = chunker_init(fd, window_size, chunk_mask, min_size, seed & 0xffffffff) * * def __dealloc__(self): # <<<<<<<<<<<<<< * if self.chunker: * chunker_free(self.chunker) */ static void __pyx_pf_5attic_7chunker_8chunkify_2__dealloc__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self) { __Pyx_RefNannyDeclarations int __pyx_t_1; __Pyx_RefNannySetupContext("__dealloc__", 0); /* "attic/chunker.pyx":24 * * def __dealloc__(self): * if self.chunker: # <<<<<<<<<<<<<< * chunker_free(self.chunker) * */ __pyx_t_1 = (__pyx_v_self->chunker != 0); if (__pyx_t_1) { /* "attic/chunker.pyx":25 * def __dealloc__(self): * if self.chunker: * chunker_free(self.chunker) # <<<<<<<<<<<<<< * * def __iter__(self): */ chunker_free(__pyx_v_self->chunker); goto __pyx_L3; } __pyx_L3:; __Pyx_RefNannyFinishContext(); } /* Python wrapper */ static PyObject *__pyx_pw_5attic_7chunker_8chunkify_5__iter__(PyObject *__pyx_v_self); /*proto*/ static PyObject *__pyx_pw_5attic_7chunker_8chunkify_5__iter__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_7chunker_8chunkify_4__iter__(((struct __pyx_obj_5attic_7chunker_chunkify *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/chunker.pyx":27 * chunker_free(self.chunker) * * def __iter__(self): # <<<<<<<<<<<<<< * return self * */ static PyObject *__pyx_pf_5attic_7chunker_8chunkify_4__iter__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iter__", 0); /* "attic/chunker.pyx":28 * * def __iter__(self): * return self # <<<<<<<<<<<<<< * * def __next__(self): */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_7chunker_8chunkify_7__next__(PyObject *__pyx_v_self); /*proto*/ static PyObject *__pyx_pw_5attic_7chunker_8chunkify_7__next__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__next__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_7chunker_8chunkify_6__next__(((struct __pyx_obj_5attic_7chunker_chunkify *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/chunker.pyx":30 * return self * * def __next__(self): # <<<<<<<<<<<<<< * return chunker_process(self.chunker) * */ static PyObject *__pyx_pf_5attic_7chunker_8chunkify_6__next__(struct __pyx_obj_5attic_7chunker_chunkify *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__next__", 0); /* "attic/chunker.pyx":31 * * def __next__(self): * return chunker_process(self.chunker) # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = chunker_process(__pyx_v_self->chunker); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("attic.chunker.chunkify.__next__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_7chunker_1buzhash(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyMethodDef __pyx_mdef_5attic_7chunker_1buzhash = {__Pyx_NAMESTR("buzhash"), (PyCFunction)__pyx_pw_5attic_7chunker_1buzhash, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}; static PyObject *__pyx_pw_5attic_7chunker_1buzhash(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { unsigned char *__pyx_v_data; unsigned long __pyx_v_seed; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("buzhash (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__data,&__pyx_n_s__seed,0}; PyObject* values[2] = {0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__data)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__seed)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("buzhash", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "buzhash") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; } else { values[0] = PyTuple_GET_ITEM(__pyx_args, 0); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); } __pyx_v_data = __Pyx_PyObject_AsUString(values[0]); if (unlikely((!__pyx_v_data) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_v_seed = __Pyx_PyInt_AsUnsignedLong(values[1]); if (unlikely((__pyx_v_seed == (unsigned long)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("buzhash", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.chunker.buzhash", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_7chunker_buzhash(__pyx_self, __pyx_v_data, __pyx_v_seed); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/chunker.pyx":34 * * * def buzhash(unsigned char *data, unsigned long seed): # <<<<<<<<<<<<<< * cdef uint32_t *table * cdef uint32_t sum */ static PyObject *__pyx_pf_5attic_7chunker_buzhash(CYTHON_UNUSED PyObject *__pyx_self, unsigned char *__pyx_v_data, unsigned long __pyx_v_seed) { uint32_t *__pyx_v_table; uint32_t __pyx_v_sum; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; uint32_t __pyx_t_3; size_t __pyx_t_4; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("buzhash", 0); /* "attic/chunker.pyx":37 * cdef uint32_t *table * cdef uint32_t sum * table = buzhash_init_table(seed & 0xffffffff) # <<<<<<<<<<<<<< * sum = c_buzhash(data, len(data), table) * free(table) */ __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_seed); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyNumber_And(__pyx_t_1, __pyx_int_4294967295); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_3 = __Pyx_PyInt_from_py_uint32_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (uint32_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_table = buzhash_init_table(__pyx_t_3); /* "attic/chunker.pyx":38 * cdef uint32_t sum * table = buzhash_init_table(seed & 0xffffffff) * sum = c_buzhash(data, len(data), table) # <<<<<<<<<<<<<< * free(table) * return sum */ __pyx_t_4 = strlen(((char *)__pyx_v_data)); __pyx_v_sum = buzhash(__pyx_v_data, __pyx_t_4, __pyx_v_table); /* "attic/chunker.pyx":39 * table = buzhash_init_table(seed & 0xffffffff) * sum = c_buzhash(data, len(data), table) * free(table) # <<<<<<<<<<<<<< * return sum * */ free(__pyx_v_table); /* "attic/chunker.pyx":40 * sum = c_buzhash(data, len(data), table) * free(table) * return sum # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); __pyx_t_2 = __Pyx_PyInt_to_py_uint32_t(__pyx_v_sum); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.chunker.buzhash", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_7chunker_3buzhash_update(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyMethodDef __pyx_mdef_5attic_7chunker_3buzhash_update = {__Pyx_NAMESTR("buzhash_update"), (PyCFunction)__pyx_pw_5attic_7chunker_3buzhash_update, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}; static PyObject *__pyx_pw_5attic_7chunker_3buzhash_update(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { uint32_t __pyx_v_sum; unsigned char __pyx_v_remove; unsigned char __pyx_v_add; size_t __pyx_v_len; unsigned long __pyx_v_seed; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("buzhash_update (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__sum,&__pyx_n_s__remove,&__pyx_n_s__add,&__pyx_n_s__len,&__pyx_n_s__seed,0}; PyObject* values[5] = {0,0,0,0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__sum)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__remove)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("buzhash_update", 1, 5, 5, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__add)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("buzhash_update", 1, 5, 5, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__len)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("buzhash_update", 1, 5, 5, 3); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 4: if (likely((values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__seed)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("buzhash_update", 1, 5, 5, 4); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "buzhash_update") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 5) { goto __pyx_L5_argtuple_error; } else { values[0] = PyTuple_GET_ITEM(__pyx_args, 0); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[3] = PyTuple_GET_ITEM(__pyx_args, 3); values[4] = PyTuple_GET_ITEM(__pyx_args, 4); } __pyx_v_sum = __Pyx_PyInt_from_py_uint32_t(values[0]); if (unlikely((__pyx_v_sum == (uint32_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_v_remove = __Pyx_PyInt_AsUnsignedChar(values[1]); if (unlikely((__pyx_v_remove == (unsigned char)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_v_add = __Pyx_PyInt_AsUnsignedChar(values[2]); if (unlikely((__pyx_v_add == (unsigned char)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_v_len = __Pyx_PyInt_AsSize_t(values[3]); if (unlikely((__pyx_v_len == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_v_seed = __Pyx_PyInt_AsUnsignedLong(values[4]); if (unlikely((__pyx_v_seed == (unsigned long)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("buzhash_update", 1, 5, 5, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.chunker.buzhash_update", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_7chunker_2buzhash_update(__pyx_self, __pyx_v_sum, __pyx_v_remove, __pyx_v_add, __pyx_v_len, __pyx_v_seed); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/chunker.pyx":43 * * * def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, unsigned long seed): # <<<<<<<<<<<<<< * cdef uint32_t *table * table = buzhash_init_table(seed & 0xffffffff) */ static PyObject *__pyx_pf_5attic_7chunker_2buzhash_update(CYTHON_UNUSED PyObject *__pyx_self, uint32_t __pyx_v_sum, unsigned char __pyx_v_remove, unsigned char __pyx_v_add, size_t __pyx_v_len, unsigned long __pyx_v_seed) { uint32_t *__pyx_v_table; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; uint32_t __pyx_t_3; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("buzhash_update", 0); /* "attic/chunker.pyx":45 * def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, unsigned long seed): * cdef uint32_t *table * table = buzhash_init_table(seed & 0xffffffff) # <<<<<<<<<<<<<< * sum = c_buzhash_update(sum, remove, add, len, table) * free(table) */ __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_seed); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = PyNumber_And(__pyx_t_1, __pyx_int_4294967295); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_3 = __Pyx_PyInt_from_py_uint32_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (uint32_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_table = buzhash_init_table(__pyx_t_3); /* "attic/chunker.pyx":46 * cdef uint32_t *table * table = buzhash_init_table(seed & 0xffffffff) * sum = c_buzhash_update(sum, remove, add, len, table) # <<<<<<<<<<<<<< * free(table) * return sum */ __pyx_v_sum = buzhash_update(__pyx_v_sum, __pyx_v_remove, __pyx_v_add, __pyx_v_len, __pyx_v_table); /* "attic/chunker.pyx":47 * table = buzhash_init_table(seed & 0xffffffff) * sum = c_buzhash_update(sum, remove, add, len, table) * free(table) # <<<<<<<<<<<<<< * return sum */ free(__pyx_v_table); /* "attic/chunker.pyx":48 * sum = c_buzhash_update(sum, remove, add, len, table) * free(table) * return sum # <<<<<<<<<<<<<< */ __Pyx_XDECREF(__pyx_r); __pyx_t_2 = __Pyx_PyInt_to_py_uint32_t(__pyx_v_sum); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.chunker.buzhash_update", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_tp_new_5attic_7chunker_chunkify(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o; o = (*t->tp_alloc)(t, 0); if (unlikely(!o)) return 0; if (unlikely(__pyx_pw_5attic_7chunker_8chunkify_1__cinit__(o, a, k) < 0)) { Py_DECREF(o); o = 0; } return o; } static void __pyx_tp_dealloc_5attic_7chunker_chunkify(PyObject *o) { { PyObject *etype, *eval, *etb; PyErr_Fetch(&etype, &eval, &etb); ++Py_REFCNT(o); __pyx_pw_5attic_7chunker_8chunkify_3__dealloc__(o); if (PyErr_Occurred()) PyErr_WriteUnraisable(o); --Py_REFCNT(o); PyErr_Restore(etype, eval, etb); } (*Py_TYPE(o)->tp_free)(o); } static PyMethodDef __pyx_methods_5attic_7chunker_chunkify[] = { {__Pyx_NAMESTR("__next__"), (PyCFunction)__pyx_pw_5attic_7chunker_8chunkify_7__next__, METH_NOARGS|METH_COEXIST, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; static PyTypeObject __pyx_type_5attic_7chunker_chunkify = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("attic.chunker.chunkify"), /*tp_name*/ sizeof(struct __pyx_obj_5attic_7chunker_chunkify), /*tp_basicsize*/ 0, /*tp_itemsize*/ __pyx_tp_dealloc_5attic_7chunker_chunkify, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else 0, /*reserved*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ __pyx_pw_5attic_7chunker_8chunkify_5__iter__, /*tp_iter*/ __pyx_pw_5attic_7chunker_8chunkify_7__next__, /*tp_iternext*/ __pyx_methods_5attic_7chunker_chunkify, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ __pyx_tp_new_5attic_7chunker_chunkify, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ 0, /*tp_del*/ #if PY_VERSION_HEX >= 0x02060000 0, /*tp_version_tag*/ #endif }; static PyMethodDef __pyx_methods[] = { {0, 0, 0, 0} }; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef __pyx_moduledef = { #if PY_VERSION_HEX < 0x03020000 { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, #else PyModuleDef_HEAD_INIT, #endif __Pyx_NAMESTR("chunker"), 0, /* m_doc */ -1, /* m_size */ __pyx_methods /* m_methods */, NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL /* m_free */ }; #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 1, 0}, {&__pyx_n_s_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 1, 1}, {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1}, {&__pyx_n_s____test__, __pyx_k____test__, sizeof(__pyx_k____test__), 0, 0, 1, 1}, {&__pyx_n_s__add, __pyx_k__add, sizeof(__pyx_k__add), 0, 0, 1, 1}, {&__pyx_n_s__buzhash, __pyx_k__buzhash, sizeof(__pyx_k__buzhash), 0, 0, 1, 1}, {&__pyx_n_s__buzhash_update, __pyx_k__buzhash_update, sizeof(__pyx_k__buzhash_update), 0, 0, 1, 1}, {&__pyx_n_s__chunk_mask, __pyx_k__chunk_mask, sizeof(__pyx_k__chunk_mask), 0, 0, 1, 1}, {&__pyx_n_s__data, __pyx_k__data, sizeof(__pyx_k__data), 0, 0, 1, 1}, {&__pyx_n_s__fd, __pyx_k__fd, sizeof(__pyx_k__fd), 0, 0, 1, 1}, {&__pyx_n_s__len, __pyx_k__len, sizeof(__pyx_k__len), 0, 0, 1, 1}, {&__pyx_n_s__min_size, __pyx_k__min_size, sizeof(__pyx_k__min_size), 0, 0, 1, 1}, {&__pyx_n_s__remove, __pyx_k__remove, sizeof(__pyx_k__remove), 0, 0, 1, 1}, {&__pyx_n_s__seed, __pyx_k__seed, sizeof(__pyx_k__seed), 0, 0, 1, 1}, {&__pyx_n_s__sum, __pyx_k__sum, sizeof(__pyx_k__sum), 0, 0, 1, 1}, {&__pyx_n_s__table, __pyx_k__table, sizeof(__pyx_k__table), 0, 0, 1, 1}, {&__pyx_n_s__window_size, __pyx_k__window_size, sizeof(__pyx_k__window_size), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { return 0; } static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); /* "attic/chunker.pyx":34 * * * def buzhash(unsigned char *data, unsigned long seed): # <<<<<<<<<<<<<< * cdef uint32_t *table * cdef uint32_t sum */ __pyx_k_tuple_1 = PyTuple_Pack(4, ((PyObject *)__pyx_n_s__data), ((PyObject *)__pyx_n_s__seed), ((PyObject *)__pyx_n_s__table), ((PyObject *)__pyx_n_s__sum)); if (unlikely(!__pyx_k_tuple_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_1); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_1)); __pyx_k_codeobj_2 = (PyObject*)__Pyx_PyCode_New(2, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_1, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_3, __pyx_n_s__buzhash, 34, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} /* "attic/chunker.pyx":43 * * * def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, unsigned long seed): # <<<<<<<<<<<<<< * cdef uint32_t *table * table = buzhash_init_table(seed & 0xffffffff) */ __pyx_k_tuple_5 = PyTuple_Pack(6, ((PyObject *)__pyx_n_s__sum), ((PyObject *)__pyx_n_s__remove), ((PyObject *)__pyx_n_s__add), ((PyObject *)__pyx_n_s__len), ((PyObject *)__pyx_n_s__seed), ((PyObject *)__pyx_n_s__table)); if (unlikely(!__pyx_k_tuple_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_5); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_5)); __pyx_k_codeobj_6 = (PyObject*)__Pyx_PyCode_New(5, 0, 6, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_5, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_3, __pyx_n_s__buzhash_update, 43, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; __Pyx_RefNannyFinishContext(); return -1; } static int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __pyx_int_4294967295 = PyInt_FromString((char *)"4294967295", 0, 0); if (unlikely(!__pyx_int_4294967295)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; return 0; __pyx_L1_error:; return -1; } #if PY_MAJOR_VERSION < 3 PyMODINIT_FUNC initchunker(void); /*proto*/ PyMODINIT_FUNC initchunker(void) #else PyMODINIT_FUNC PyInit_chunker(void); /*proto*/ PyMODINIT_FUNC PyInit_chunker(void) #endif { PyObject *__pyx_t_1 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannyDeclarations #if CYTHON_REFNANNY __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); if (!__Pyx_RefNanny) { PyErr_Clear(); __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); if (!__Pyx_RefNanny) Py_FatalError("failed to import 'refnanny' module"); } #endif __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_chunker(void)", 0); if ( __Pyx_check_binary_version() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #ifdef __Pyx_CyFunction_USED if (__Pyx_CyFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif #ifdef __Pyx_FusedFunction_USED if (__pyx_FusedFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif #ifdef __Pyx_Generator_USED if (__pyx_Generator_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif /*--- Library function declarations ---*/ /*--- Threads initialization code ---*/ #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS #ifdef WITH_THREAD /* Python build with threading support? */ PyEval_InitThreads(); #endif #endif /*--- Module creation code ---*/ #if PY_MAJOR_VERSION < 3 __pyx_m = Py_InitModule4(__Pyx_NAMESTR("chunker"), __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); #else __pyx_m = PyModule_Create(&__pyx_moduledef); #endif if (unlikely(!__pyx_m)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} Py_INCREF(__pyx_d); #if PY_MAJOR_VERSION >= 3 { PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (!PyDict_GetItemString(modules, "attic.chunker")) { if (unlikely(PyDict_SetItemString(modules, "attic.chunker", __pyx_m) < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } } #endif __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (unlikely(!__pyx_b)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #if CYTHON_COMPILING_IN_PYPY Py_INCREF(__pyx_b); #endif if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; /*--- Initialize various global constants etc. ---*/ if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) if (__Pyx_init_sys_getdefaultencoding_params() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif if (__pyx_module_is_main_attic__chunker) { if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_n_s____main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} /*--- Constants init code ---*/ if (unlikely(__Pyx_InitCachedConstants() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} /*--- Global init code ---*/ /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ if (PyType_Ready(&__pyx_type_5attic_7chunker_chunkify) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "chunkify", (PyObject *)&__pyx_type_5attic_7chunker_chunkify) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5attic_7chunker_chunkify = &__pyx_type_5attic_7chunker_chunkify; /*--- Type import code ---*/ /*--- Variable import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ /* "attic/chunker.pyx":34 * * * def buzhash(unsigned char *data, unsigned long seed): # <<<<<<<<<<<<<< * cdef uint32_t *table * cdef uint32_t sum */ __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5attic_7chunker_1buzhash, NULL, __pyx_n_s_4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s__buzhash, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "attic/chunker.pyx":43 * * * def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, unsigned long seed): # <<<<<<<<<<<<<< * cdef uint32_t *table * table = buzhash_init_table(seed & 0xffffffff) */ __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5attic_7chunker_3buzhash_update, NULL, __pyx_n_s_4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s__buzhash_update, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "attic/chunker.pyx":1 * # -*- coding: utf-8 -*- # <<<<<<<<<<<<<< * * from libc.stdlib cimport free */ __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); if (PyDict_SetItem(__pyx_d, __pyx_n_s____test__, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); if (__pyx_m) { __Pyx_AddTraceback("init attic.chunker", __pyx_clineno, __pyx_lineno, __pyx_filename); Py_DECREF(__pyx_m); __pyx_m = 0; } else if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ImportError, "init attic.chunker"); } __pyx_L0:; __Pyx_RefNannyFinishContext(); #if PY_MAJOR_VERSION < 3 return; #else return __pyx_m; #endif } /* Runtime support code */ #if CYTHON_REFNANNY static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { PyObject *m = NULL, *p = NULL; void *r = NULL; m = PyImport_ImportModule((char *)modname); if (!m) goto end; p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); if (!p) goto end; r = PyLong_AsVoidPtr(p); end: Py_XDECREF(p); Py_XDECREF(m); return (__Pyx_RefNannyAPIStruct *)r; } #endif /* CYTHON_REFNANNY */ static void __Pyx_RaiseArgtupleInvalid( const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found) { Py_ssize_t num_expected; const char *more_or_less; if (num_found < num_min) { num_expected = num_min; more_or_less = "at least"; } else { num_expected = num_max; more_or_less = "at most"; } if (exact) { more_or_less = "exactly"; } PyErr_Format(PyExc_TypeError, "%s() takes %s %" CYTHON_FORMAT_SSIZE_T "d positional argument%s (%" CYTHON_FORMAT_SSIZE_T "d given)", func_name, more_or_less, num_expected, (num_expected == 1) ? "" : "s", num_found); } static void __Pyx_RaiseDoubleKeywordsError( const char* func_name, PyObject* kw_name) { PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION >= 3 "%s() got multiple values for keyword argument '%U'", func_name, kw_name); #else "%s() got multiple values for keyword argument '%s'", func_name, PyString_AsString(kw_name)); #endif } static int __Pyx_ParseOptionalKeywords( PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name) { PyObject *key = 0, *value = 0; Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { values[name-argnames] = value; continue; } name = first_kw_arg; #if PY_MAJOR_VERSION < 3 if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { while (*name) { if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) && _PyString_Eq(**name, key)) { values[name-argnames] = value; break; } name++; } if (*name) continue; else { PyObject*** argname = argnames; while (argname != first_kw_arg) { if ((**argname == key) || ( (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) && _PyString_Eq(**argname, key))) { goto arg_passed_twice; } argname++; } } } else #endif if (likely(PyUnicode_Check(key))) { while (*name) { int cmp = (**name == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : #endif PyUnicode_Compare(**name, key); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; break; } name++; } if (*name) continue; else { PyObject*** argname = argnames; while (argname != first_kw_arg) { int cmp = (**argname == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : #endif PyUnicode_Compare(**argname, key); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) goto arg_passed_twice; argname++; } } } else goto invalid_keyword_type; if (kwds2) { if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; } else { goto invalid_keyword; } } return 0; arg_passed_twice: __Pyx_RaiseDoubleKeywordsError(function_name, key); goto bad; invalid_keyword_type: PyErr_Format(PyExc_TypeError, "%s() keywords must be strings", function_name); goto bad; invalid_keyword: PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 "%s() got an unexpected keyword argument '%s'", function_name, PyString_AsString(key)); #else "%s() got an unexpected keyword argument '%U'", function_name, key); #endif bad: return -1; } static CYTHON_INLINE uint32_t __Pyx_PyInt_from_py_uint32_t(PyObject* x) { const uint32_t neg_one = (uint32_t)-1, const_zero = (uint32_t)0; const int is_unsigned = const_zero < neg_one; if (sizeof(uint32_t) == sizeof(char)) { if (is_unsigned) return (uint32_t)__Pyx_PyInt_AsUnsignedChar(x); else return (uint32_t)__Pyx_PyInt_AsSignedChar(x); } else if (sizeof(uint32_t) == sizeof(short)) { if (is_unsigned) return (uint32_t)__Pyx_PyInt_AsUnsignedShort(x); else return (uint32_t)__Pyx_PyInt_AsSignedShort(x); } else if (sizeof(uint32_t) == sizeof(int)) { if (is_unsigned) return (uint32_t)__Pyx_PyInt_AsUnsignedInt(x); else return (uint32_t)__Pyx_PyInt_AsSignedInt(x); } else if (sizeof(uint32_t) == sizeof(long)) { if (is_unsigned) return (uint32_t)__Pyx_PyInt_AsUnsignedLong(x); else return (uint32_t)__Pyx_PyInt_AsSignedLong(x); } else if (sizeof(uint32_t) == sizeof(PY_LONG_LONG)) { if (is_unsigned) return (uint32_t)__Pyx_PyInt_AsUnsignedLongLong(x); else return (uint32_t)__Pyx_PyInt_AsSignedLongLong(x); } else { #if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else uint32_t val; PyObject *v = __Pyx_PyNumber_Int(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { PyObject *tmp = v; v = PyNumber_Long(tmp); Py_DECREF(tmp); } #endif if (likely(v)) { int one = 1; int is_little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&val; int ret = _PyLong_AsByteArray((PyLongObject *)v, bytes, sizeof(val), is_little, !is_unsigned); Py_DECREF(v); if (likely(!ret)) return val; } #endif return (uint32_t)-1; } } static CYTHON_INLINE PyObject *__Pyx_PyInt_to_py_uint32_t(uint32_t val) { const uint32_t neg_one = (uint32_t)-1, const_zero = (uint32_t)0; const int is_unsigned = const_zero < neg_one; if ((sizeof(uint32_t) == sizeof(char)) || (sizeof(uint32_t) == sizeof(short))) { return PyInt_FromLong((long)val); } else if ((sizeof(uint32_t) == sizeof(int)) || (sizeof(uint32_t) == sizeof(long))) { if (is_unsigned) return PyLong_FromUnsignedLong((unsigned long)val); else return PyInt_FromLong((long)val); } else if (sizeof(uint32_t) == sizeof(PY_LONG_LONG)) { if (is_unsigned) return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)val); else return PyLong_FromLongLong((PY_LONG_LONG)val); } else { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&val; return _PyLong_FromByteArray(bytes, sizeof(uint32_t), little, !is_unsigned); } } static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { const unsigned char neg_one = (unsigned char)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(unsigned char) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(unsigned char)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to unsigned char" : "value too large to convert to unsigned char"); } return (unsigned char)-1; } return (unsigned char)val; } return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); } static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { const unsigned short neg_one = (unsigned short)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(unsigned short) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(unsigned short)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to unsigned short" : "value too large to convert to unsigned short"); } return (unsigned short)-1; } return (unsigned short)val; } return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); } static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { const unsigned int neg_one = (unsigned int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(unsigned int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(unsigned int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to unsigned int" : "value too large to convert to unsigned int"); } return (unsigned int)-1; } return (unsigned int)val; } return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); } static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject* x) { const char neg_one = (char)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(char) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(char)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to char" : "value too large to convert to char"); } return (char)-1; } return (char)val; } return (char)__Pyx_PyInt_AsLong(x); } static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject* x) { const short neg_one = (short)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(short) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(short)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to short" : "value too large to convert to short"); } return (short)-1; } return (short)val; } return (short)__Pyx_PyInt_AsLong(x); } static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject* x) { const int neg_one = (int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to int" : "value too large to convert to int"); } return (int)-1; } return (int)val; } return (int)__Pyx_PyInt_AsLong(x); } static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { const signed char neg_one = (signed char)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(signed char) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(signed char)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to signed char" : "value too large to convert to signed char"); } return (signed char)-1; } return (signed char)val; } return (signed char)__Pyx_PyInt_AsSignedLong(x); } static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { const signed short neg_one = (signed short)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(signed short) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(signed short)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to signed short" : "value too large to convert to signed short"); } return (signed short)-1; } return (signed short)val; } return (signed short)__Pyx_PyInt_AsSignedLong(x); } static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { const signed int neg_one = (signed int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(signed int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(signed int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to signed int" : "value too large to convert to signed int"); } return (signed int)-1; } return (signed int)val; } return (signed int)__Pyx_PyInt_AsSignedLong(x); } static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject* x) { const int neg_one = (int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to int" : "value too large to convert to int"); } return (int)-1; } return (int)val; } return (int)__Pyx_PyInt_AsLong(x); } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { const unsigned long neg_one = (unsigned long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned long"); return (unsigned long)-1; } return (unsigned long)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (unsigned long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned long"); return (unsigned long)-1; } return (unsigned long)PyLong_AsUnsignedLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(unsigned long) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(unsigned long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (unsigned long)PyLong_AsLong(x); } } else { unsigned long val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (unsigned long)-1; val = __Pyx_PyInt_AsUnsignedLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { const unsigned PY_LONG_LONG neg_one = (unsigned PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned PY_LONG_LONG"); return (unsigned PY_LONG_LONG)-1; } return (unsigned PY_LONG_LONG)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned PY_LONG_LONG"); return (unsigned PY_LONG_LONG)-1; } return (unsigned PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (unsigned PY_LONG_LONG)PyLong_AsLongLong(x); } } else { unsigned PY_LONG_LONG val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (unsigned PY_LONG_LONG)-1; val = __Pyx_PyInt_AsUnsignedLongLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { const long neg_one = (long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to long"); return (long)-1; } return (long)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to long"); return (long)-1; } return (long)PyLong_AsUnsignedLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(long) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (long)PyLong_AsLong(x); } } else { long val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (long)-1; val = __Pyx_PyInt_AsLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { const PY_LONG_LONG neg_one = (PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to PY_LONG_LONG"); return (PY_LONG_LONG)-1; } return (PY_LONG_LONG)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to PY_LONG_LONG"); return (PY_LONG_LONG)-1; } return (PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (PY_LONG_LONG)PyLong_AsLongLong(x); } } else { PY_LONG_LONG val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (PY_LONG_LONG)-1; val = __Pyx_PyInt_AsLongLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { const signed long neg_one = (signed long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed long"); return (signed long)-1; } return (signed long)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (signed long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed long"); return (signed long)-1; } return (signed long)PyLong_AsUnsignedLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(signed long) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(signed long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (signed long)PyLong_AsLong(x); } } else { signed long val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (signed long)-1; val = __Pyx_PyInt_AsSignedLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { const signed PY_LONG_LONG neg_one = (signed PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed PY_LONG_LONG"); return (signed PY_LONG_LONG)-1; } return (signed PY_LONG_LONG)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed PY_LONG_LONG"); return (signed PY_LONG_LONG)-1; } return (signed PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (signed PY_LONG_LONG)PyLong_AsLongLong(x); } } else { signed PY_LONG_LONG val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (signed PY_LONG_LONG)-1; val = __Pyx_PyInt_AsSignedLongLong(tmp); Py_DECREF(tmp); return val; } } static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { char message[200]; PyOS_snprintf(message, sizeof(message), "compiletime version %s of module '%.100s' " "does not match runtime version %s", ctversion, __Pyx_MODULE_NAME, rtversion); #if PY_VERSION_HEX < 0x02050000 return PyErr_Warn(NULL, message); #else return PyErr_WarnEx(NULL, message, 1); #endif } return 0; } static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { return count; } while (start < end) { mid = (start + end) / 2; if (code_line < entries[mid].code_line) { end = mid; } else if (code_line > entries[mid].code_line) { start = mid + 1; } else { return mid; } } if (code_line <= entries[mid].code_line) { return mid; } else { return mid + 1; } } static PyCodeObject *__pyx_find_code_object(int code_line) { PyCodeObject* code_object; int pos; if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { return NULL; } pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { return NULL; } code_object = __pyx_code_cache.entries[pos].code_object; Py_INCREF(code_object); return code_object; } static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { int pos, i; __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; if (unlikely(!code_line)) { return; } if (unlikely(!entries)) { entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); if (likely(entries)) { __pyx_code_cache.entries = entries; __pyx_code_cache.max_count = 64; __pyx_code_cache.count = 1; entries[0].code_line = code_line; entries[0].code_object = code_object; Py_INCREF(code_object); } return; } pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { PyCodeObject* tmp = entries[pos].code_object; entries[pos].code_object = code_object; Py_DECREF(tmp); return; } if (__pyx_code_cache.count == __pyx_code_cache.max_count) { int new_max = __pyx_code_cache.max_count + 64; entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( __pyx_code_cache.entries, new_max*sizeof(__Pyx_CodeObjectCacheEntry)); if (unlikely(!entries)) { return; } __pyx_code_cache.entries = entries; __pyx_code_cache.max_count = new_max; } for (i=__pyx_code_cache.count; i>pos; i--) { entries[i] = entries[i-1]; } entries[pos].code_line = code_line; entries[pos].code_object = code_object; __pyx_code_cache.count++; Py_INCREF(code_object); } #include "compile.h" #include "frameobject.h" #include "traceback.h" static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyObject *py_srcfile = 0; PyObject *py_funcname = 0; #if PY_MAJOR_VERSION < 3 py_srcfile = PyString_FromString(filename); #else py_srcfile = PyUnicode_FromString(filename); #endif if (!py_srcfile) goto bad; if (c_line) { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); #else py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); #endif } else { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromString(funcname); #else py_funcname = PyUnicode_FromString(funcname); #endif } if (!py_funcname) goto bad; py_code = __Pyx_PyCode_New( 0, /*int argcount,*/ 0, /*int kwonlyargcount,*/ 0, /*int nlocals,*/ 0, /*int stacksize,*/ 0, /*int flags,*/ __pyx_empty_bytes, /*PyObject *code,*/ __pyx_empty_tuple, /*PyObject *consts,*/ __pyx_empty_tuple, /*PyObject *names,*/ __pyx_empty_tuple, /*PyObject *varnames,*/ __pyx_empty_tuple, /*PyObject *freevars,*/ __pyx_empty_tuple, /*PyObject *cellvars,*/ py_srcfile, /*PyObject *filename,*/ py_funcname, /*PyObject *name,*/ py_line, /*int firstlineno,*/ __pyx_empty_bytes /*PyObject *lnotab*/ ); Py_DECREF(py_srcfile); Py_DECREF(py_funcname); return py_code; bad: Py_XDECREF(py_srcfile); Py_XDECREF(py_funcname); return NULL; } static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyObject *py_globals = 0; PyFrameObject *py_frame = 0; py_code = __pyx_find_code_object(c_line ? c_line : py_line); if (!py_code) { py_code = __Pyx_CreateCodeObjectForTraceback( funcname, c_line, py_line, filename); if (!py_code) goto bad; __pyx_insert_code_object(c_line ? c_line : py_line, py_code); } py_globals = PyModule_GetDict(__pyx_m); if (!py_globals) goto bad; py_frame = PyFrame_New( PyThreadState_GET(), /*PyThreadState *tstate,*/ py_code, /*PyCodeObject *code,*/ py_globals, /*PyObject *globals,*/ 0 /*PyObject *locals*/ ); if (!py_frame) goto bad; py_frame->f_lineno = py_line; PyTraceBack_Here(py_frame); bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { *t->p = PyString_InternFromString(t->s); } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } #else /* Python 3+ has unicode identifiers */ if (t->is_unicode | t->is_str) { if (t->intern) { *t->p = PyUnicode_InternFromString(t->s); } else if (t->encoding) { *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); } else { *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); } } else { *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); } #endif if (!*t->p) return -1; ++t; } return 0; } static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char* c_str) { return __Pyx_PyUnicode_FromStringAndSize(c_str, strlen(c_str)); } static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { Py_ssize_t ignore; return __Pyx_PyObject_AsStringAndSize(o, &ignore); } static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT if ( #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII __Pyx_sys_getdefaultencoding_not_ascii && #endif PyUnicode_Check(o)) { #if PY_VERSION_HEX < 0x03030000 char* defenc_c; PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); if (!defenc) return NULL; defenc_c = PyBytes_AS_STRING(defenc); #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII { char* end = defenc_c + PyBytes_GET_SIZE(defenc); char* c; for (c = defenc_c; c < end; c++) { if ((unsigned char) (*c) >= 128) { PyUnicode_AsASCIIString(o); return NULL; } } } #endif /*__PYX_DEFAULT_STRING_ENCODING_IS_ASCII*/ *length = PyBytes_GET_SIZE(defenc); return defenc_c; #else /* PY_VERSION_HEX < 0x03030000 */ if (PyUnicode_READY(o) == -1) return NULL; #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII if (PyUnicode_IS_ASCII(o)) { *length = PyUnicode_GET_DATA_SIZE(o); return PyUnicode_AsUTF8(o); } else { PyUnicode_AsASCIIString(o); return NULL; } #else /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ return PyUnicode_AsUTF8AndSize(o, length); #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ #endif /* PY_VERSION_HEX < 0x03030000 */ } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ { char* result; int r = PyBytes_AsStringAndSize(o, &result, length); if (r < 0) { return NULL; } else { return result; } } } static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { int is_true = x == Py_True; if (is_true | (x == Py_False) | (x == Py_None)) return is_true; else return PyObject_IsTrue(x); } static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { PyNumberMethods *m; const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 if (PyInt_Check(x) || PyLong_Check(x)) #else if (PyLong_Check(x)) #endif return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } else if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } #else if (m && m->nb_int) { name = "int"; res = PyNumber_Long(x); } #endif if (res) { #if PY_MAJOR_VERSION < 3 if (!PyInt_Check(res) && !PyLong_Check(res)) { #else if (!PyLong_Check(res)) { #endif PyErr_Format(PyExc_TypeError, "__%s__ returned non-%s (type %.200s)", name, name, Py_TYPE(res)->tp_name); Py_DECREF(res); return NULL; } } else if (!PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "an integer is required"); } return res; } static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { Py_ssize_t ival; PyObject* x = PyNumber_Index(b); if (!x) return -1; ival = PyInt_AsSsize_t(x); Py_DECREF(x); return ival; } static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { #if PY_VERSION_HEX < 0x02050000 if (ival <= LONG_MAX) return PyInt_FromLong((long)ival); else { unsigned char *bytes = (unsigned char *) &ival; int one = 1; int little = (int)*(unsigned char*)&one; return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); } #else return PyInt_FromSize_t(ival); #endif } static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { if ((val != (unsigned PY_LONG_LONG)-1) || !PyErr_Occurred()) PyErr_SetString(PyExc_OverflowError, "value too large to convert to size_t"); return (size_t)-1; } return (size_t)val; } #endif /* Py_PYTHON_H */ Attic-0.10/attic/_chunker.c0000644000175000017500000001676712272013203016023 0ustar jonasjonas00000000000000#include /* Cyclic polynomial / buzhash: https://en.wikipedia.org/wiki/Rolling_hash */ static uint32_t table_base[] = { 0xe7f831ec, 0xf4026465, 0xafb50cae, 0x6d553c7a, 0xd639efe3, 0x19a7b895, 0x9aba5b21, 0x5417d6d4, 0x35fd2b84, 0xd1f6a159, 0x3f8e323f, 0xb419551c, 0xf444cebf, 0x21dc3b80, 0xde8d1e36, 0x84a32436, 0xbeb35a9d, 0xa36f24aa, 0xa4e60186, 0x98d18ffe, 0x3f042f9e, 0xdb228bcd, 0x096474b7, 0x5c20c2f7, 0xf9eec872, 0xe8625275, 0xb9d38f80, 0xd48eb716, 0x22a950b4, 0x3cbaaeaa, 0xc37cddd3, 0x8fea6f6a, 0x1d55d526, 0x7fd6d3b3, 0xdaa072ee, 0x4345ac40, 0xa077c642, 0x8f2bd45b, 0x28509110, 0x55557613, 0xffc17311, 0xd961ffef, 0xe532c287, 0xaab95937, 0x46d38365, 0xb065c703, 0xf2d91d0f, 0x92cd4bb0, 0x4007c712, 0xf35509dd, 0x505b2f69, 0x557ead81, 0x310f4563, 0xbddc5be8, 0x9760f38c, 0x701e0205, 0x00157244, 0x14912826, 0xdc4ca32b, 0x67b196de, 0x5db292e8, 0x8c1b406b, 0x01f34075, 0xfa2520f7, 0x73bc37ab, 0x1e18bc30, 0xfe2c6cb3, 0x20c522d0, 0x5639e3db, 0x942bda35, 0x899af9d1, 0xced44035, 0x98cc025b, 0x255f5771, 0x70fefa24, 0xe928fa4d, 0x2c030405, 0xb9325590, 0x20cb63bd, 0xa166305d, 0x80e52c0a, 0xa8fafe2f, 0x1ad13f7d, 0xcfaf3685, 0x6c83a199, 0x7d26718a, 0xde5dfcd9, 0x79cf7355, 0x8979d7fb, 0xebf8c55e, 0xebe408e4, 0xcd2affba, 0xe483be6e, 0xe239d6de, 0x5dc1e9e0, 0x0473931f, 0x851b097c, 0xac5db249, 0x09c0f9f2, 0xd8d2f134, 0xe6f38e41, 0xb1c71bf1, 0x52b6e4db, 0x07224424, 0x6cf73e85, 0x4f25d89c, 0x782a7d74, 0x10a68dcd, 0x3a868189, 0xd570d2dc, 0x69630745, 0x9542ed86, 0x331cd6b2, 0xa84b5b28, 0x07879c9d, 0x38372f64, 0x7185db11, 0x25ba7c83, 0x01061523, 0xe6792f9f, 0xe5df07d1, 0x4321b47f, 0x7d2469d8, 0x1a3a4f90, 0x48be29a3, 0x669071af, 0x8ec8dd31, 0x0810bfbf, 0x813a06b4, 0x68538345, 0x65865ddc, 0x43a71b8e, 0x78619a56, 0x5a34451d, 0x5bdaa3ed, 0x71edc7e9, 0x17ac9a20, 0x78d10bfa, 0x6c1e7f35, 0xd51839d9, 0x240cbc51, 0x33513cc1, 0xd2b4f795, 0xccaa8186, 0x0babe682, 0xa33cf164, 0x18c643ea, 0xc1ca105f, 0x9959147a, 0x6d3d94de, 0x0b654fbe, 0xed902ca0, 0x7d835cb5, 0x99ba1509, 0x6445c922, 0x495e76c2, 0xf07194bc, 0xa1631d7e, 0x677076a5, 0x89fffe35, 0x1a49bcf3, 0x8e6c948a, 0x0144c917, 0x8d93aea1, 0x16f87ddf, 0xc8f25d49, 0x1fb11297, 0x27e750cd, 0x2f422da1, 0xdee89a77, 0x1534c643, 0x457b7b8b, 0xaf172f7a, 0x6b9b09d6, 0x33573f7f, 0xf14e15c4, 0x526467d5, 0xaf488241, 0x87c3ee0d, 0x33be490c, 0x95aa6e52, 0x43ec242e, 0xd77de99b, 0xd018334f, 0x5b78d407, 0x498eb66b, 0xb1279fa8, 0xb38b0ea6, 0x90718376, 0xe325dee2, 0x8e2f2cba, 0xcaa5bdec, 0x9d652c56, 0xad68f5cb, 0xa77591af, 0x88e37ee8, 0xf8faa221, 0xfcbbbe47, 0x4f407786, 0xaf393889, 0xf444a1d9, 0x15ae1a2f, 0x40aa7097, 0x6f9486ac, 0x29d232a3, 0xe47609e9, 0xe8b631ff, 0xba8565f4, 0x11288749, 0x46c9a838, 0xeb1b7cd8, 0xf516bbb1, 0xfb74fda0, 0x010996e6, 0x4c994653, 0x1d889512, 0x53dcd9a3, 0xdd074697, 0x1e78e17c, 0x637c98bf, 0x930bb219, 0xcf7f75b0, 0xcb9355fb, 0x9e623009, 0xe466d82c, 0x28f968d3, 0xfeb385d9, 0x238e026c, 0xb8ed0560, 0x0c6a027a, 0x3d6fec4b, 0xbb4b2ec2, 0xe715031c, 0xeded011d, 0xcdc4d3b9, 0xc456fc96, 0xdd0eea20, 0xb3df8ec9, 0x12351993, 0xd9cbb01c, 0x603147a2, 0xcf37d17d, 0xf7fcd9dc, 0xd8556fa3, 0x104c8131, 0x13152774, 0xb4715811, 0x6a72c2c9, 0xc5ae37bb, 0xa76ce12a, 0x8150d8f3, 0x2ec29218, 0xa35f0984, 0x48c0647e, 0x0b5ff98c, 0x71893f7b }; #define BARREL_SHIFT(v, shift) ( ((v) << ((shift) & 0x1f)) | ((v) >> (32 - ((shift) & 0x1f))) ) static uint32_t * buzhash_init_table(uint32_t seed) { int i; uint32_t *table = malloc(1024); for(i = 0; i < 256; i++) { table[i] = table_base[i] ^ seed; } return table; } static uint32_t buzhash(const unsigned char *data, size_t len, const uint32_t *h) { size_t i; uint32_t sum = 0; for(i = len - 1; i > 0; i--) { sum ^= BARREL_SHIFT(h[*data], i); data++; } return sum ^ h[*data]; } static uint32_t buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len, const uint32_t *h) { return BARREL_SHIFT(sum, 1) ^ BARREL_SHIFT(h[remove], len) ^ h[add]; } typedef struct { int window_size, chunk_mask, min_size; size_t buf_size; uint32_t *table; uint8_t *data; PyObject *fd; int done, eof; size_t remaining, bytes_read, bytes_yielded, position, last; } Chunker; static Chunker * chunker_init(PyObject *fd, int window_size, int chunk_mask, int min_size, uint32_t seed) { Chunker *c = malloc(sizeof(Chunker)); c->window_size = window_size; c->chunk_mask = chunk_mask; c->min_size = min_size; c->table = buzhash_init_table(seed); c->buf_size = 10 * 1024 * 1024; c->data = malloc(c->buf_size); c->fd = fd; Py_INCREF(fd); c->done = 0; c->remaining = 0; c->bytes_read = 0; c->bytes_yielded = 0; c->position = 0; c->last = 0; c->eof = 0; return c; } static void chunker_free(Chunker *c) { Py_DECREF(c->fd); free(c->table); free(c->data); free(c); } static int chunker_fill(Chunker *c) { size_t n; memmove(c->data, c->data + c->last, c->position + c->remaining - c->last); c->position -= c->last; c->last = 0; n = c->buf_size - c->position - c->remaining; if(c->eof || n == 0) { return 1; } PyObject *data = PyObject_CallMethod(c->fd, "read", "i", n); if(!data) { return 0; } n = PyBytes_Size(data); if(n) { memcpy(c->data + c->position + c->remaining, PyBytes_AsString(data), n); c->remaining += n; c->bytes_read += n; } else { c->eof = 1; } Py_DECREF(data); return 1; } PyObject * PyBuffer_FromMemory(void *data, Py_ssize_t len) { Py_buffer buffer; PyObject *mv; PyBuffer_FillInfo(&buffer, NULL, data, len, 1, PyBUF_CONTIG_RO); mv = PyMemoryView_FromBuffer(&buffer); PyBuffer_Release(&buffer); return mv; } static PyObject * chunker_process(Chunker *c) { uint32_t sum, chunk_mask = c->chunk_mask, min_size = c->min_size, window_size = c->window_size; int n = 0; if(c->done) { if(c->bytes_read == c->bytes_yielded) PyErr_SetNone(PyExc_StopIteration); else PyErr_SetString(PyExc_Exception, "chunkifier byte count mismatch"); return NULL; } if(c->remaining <= window_size) { if(!chunker_fill(c)) { return NULL; } } if(c->remaining < window_size) { c->done = 1; if(c->remaining) { c->bytes_yielded += c->remaining; return PyBuffer_FromMemory(c->data + c->position, c->remaining); } else { if(c->bytes_read == c->bytes_yielded) PyErr_SetNone(PyExc_StopIteration); else PyErr_SetString(PyExc_Exception, "chunkifier byte count mismatch"); return NULL; } } sum = buzhash(c->data + c->position, window_size, c->table); while(c->remaining > c->window_size && ((sum & chunk_mask) || n < min_size)) { sum = buzhash_update(sum, c->data[c->position], c->data[c->position + window_size], window_size, c->table); c->position++; c->remaining--; n++; if(c->remaining <= window_size) { if(!chunker_fill(c)) { return NULL; } } } if(c->remaining <= window_size) { c->position += c->remaining; c->remaining = 0; } int old_last = c->last; c->last = c->position; n = c->last - old_last; c->bytes_yielded += n; return PyBuffer_FromMemory(c->data + old_last, n); } Attic-0.10/attic/hashindex.c0000644000175000017500000067706212272541113016207 0ustar jonasjonas00000000000000/* Generated by Cython 0.19.1 on Thu Jan 30 22:21:14 2014 */ #define PY_SSIZE_T_CLEAN #ifndef CYTHON_USE_PYLONG_INTERNALS #ifdef PYLONG_BITS_IN_DIGIT #define CYTHON_USE_PYLONG_INTERNALS 0 #else #include "pyconfig.h" #ifdef PYLONG_BITS_IN_DIGIT #define CYTHON_USE_PYLONG_INTERNALS 1 #else #define CYTHON_USE_PYLONG_INTERNALS 0 #endif #endif #endif #include "Python.h" #ifndef Py_PYTHON_H #error Python headers needed to compile C extensions, please install development version of Python. #elif PY_VERSION_HEX < 0x02040000 #error Cython requires Python 2.4+. #else #include /* For offsetof */ #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif #if !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif #ifndef __cdecl #define __cdecl #endif #ifndef __fastcall #define __fastcall #endif #endif #ifndef DL_IMPORT #define DL_IMPORT(t) t #endif #ifndef DL_EXPORT #define DL_EXPORT(t) t #endif #ifndef PY_LONG_LONG #define PY_LONG_LONG LONG_LONG #endif #ifndef Py_HUGE_VAL #define Py_HUGE_VAL HUGE_VAL #endif #ifdef PYPY_VERSION #define CYTHON_COMPILING_IN_PYPY 1 #define CYTHON_COMPILING_IN_CPYTHON 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_CPYTHON 1 #endif #if PY_VERSION_HEX < 0x02050000 typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX #define PY_SSIZE_T_MIN INT_MIN #define PY_FORMAT_SIZE_T "" #define CYTHON_FORMAT_SSIZE_T "" #define PyInt_FromSsize_t(z) PyInt_FromLong(z) #define PyInt_AsSsize_t(o) __Pyx_PyInt_AsInt(o) #define PyNumber_Index(o) ((PyNumber_Check(o) && !PyFloat_Check(o)) ? PyNumber_Int(o) : \ (PyErr_Format(PyExc_TypeError, \ "expected index value, got %.200s", Py_TYPE(o)->tp_name), \ (PyObject*)0)) #define __Pyx_PyIndex_Check(o) (PyNumber_Check(o) && !PyFloat_Check(o) && \ !PyComplex_Check(o)) #define PyIndex_Check __Pyx_PyIndex_Check #define PyErr_WarnEx(category, message, stacklevel) PyErr_Warn(category, message) #define __PYX_BUILD_PY_SSIZE_T "i" #else #define __PYX_BUILD_PY_SSIZE_T "n" #define CYTHON_FORMAT_SSIZE_T "z" #define __Pyx_PyIndex_Check PyIndex_Check #endif #if PY_VERSION_HEX < 0x02060000 #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) #define PyVarObject_HEAD_INIT(type, size) \ PyObject_HEAD_INIT(type) size, #define PyType_Modified(t) typedef struct { void *buf; PyObject *obj; Py_ssize_t len; Py_ssize_t itemsize; int readonly; int ndim; char *format; Py_ssize_t *shape; Py_ssize_t *strides; Py_ssize_t *suboffsets; void *internal; } Py_buffer; #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 #define PyBUF_FORMAT 0x0004 #define PyBUF_ND 0x0008 #define PyBUF_STRIDES (0x0010 | PyBUF_ND) #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) #define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_FORMAT | PyBUF_WRITABLE) #define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_FORMAT | PyBUF_WRITABLE) typedef int (*getbufferproc)(PyObject *, Py_buffer *, int); typedef void (*releasebufferproc)(PyObject *, Py_buffer *); #endif #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #endif #if PY_MAJOR_VERSION < 3 && PY_MINOR_VERSION < 6 #define PyUnicode_FromString(s) PyUnicode_Decode(s, strlen(s), "UTF-8", "strict") #endif #if PY_MAJOR_VERSION >= 3 #define Py_TPFLAGS_CHECKTYPES 0 #define Py_TPFLAGS_HAVE_INDEX 0 #endif #if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) #define Py_TPFLAGS_HAVE_NEWBUFFER 0 #endif #if PY_VERSION_HEX < 0x02060000 #define Py_TPFLAGS_HAVE_VERSION_TAG 0 #endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #else #define CYTHON_PEP393_ENABLED 0 #define __Pyx_PyUnicode_READY(op) (0) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) #define __Pyx_PyUnicode_READ(k, d, i) ((k=k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #endif #if PY_MAJOR_VERSION >= 3 #define PyBaseString_Type PyUnicode_Type #define PyStringObject PyUnicodeObject #define PyString_Type PyUnicode_Type #define PyString_Check PyUnicode_Check #define PyString_CheckExact PyUnicode_CheckExact #endif #if PY_VERSION_HEX < 0x02060000 #define PyBytesObject PyStringObject #define PyBytes_Type PyString_Type #define PyBytes_Check PyString_Check #define PyBytes_CheckExact PyString_CheckExact #define PyBytes_FromString PyString_FromString #define PyBytes_FromStringAndSize PyString_FromStringAndSize #define PyBytes_FromFormat PyString_FromFormat #define PyBytes_DecodeEscape PyString_DecodeEscape #define PyBytes_AsString PyString_AsString #define PyBytes_AsStringAndSize PyString_AsStringAndSize #define PyBytes_Size PyString_Size #define PyBytes_AS_STRING PyString_AS_STRING #define PyBytes_GET_SIZE PyString_GET_SIZE #define PyBytes_Repr PyString_Repr #define PyBytes_Concat PyString_Concat #define PyBytes_ConcatAndDel PyString_ConcatAndDel #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) #else #define __Pyx_PyBaseString_Check(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj) || \ PyString_Check(obj) || PyUnicode_Check(obj)) #define __Pyx_PyBaseString_CheckExact(obj) (Py_TYPE(obj) == &PyBaseString_Type) #endif #if PY_VERSION_HEX < 0x02060000 #define PySet_Check(obj) PyObject_TypeCheck(obj, &PySet_Type) #define PyFrozenSet_Check(obj) PyObject_TypeCheck(obj, &PyFrozenSet_Type) #endif #ifndef PySet_CheckExact #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject #define PyInt_Type PyLong_Type #define PyInt_Check(op) PyLong_Check(op) #define PyInt_CheckExact(op) PyLong_CheckExact(op) #define PyInt_FromString PyLong_FromString #define PyInt_FromUnicode PyLong_FromUnicode #define PyInt_FromLong PyLong_FromLong #define PyInt_FromSize_t PyLong_FromSize_t #define PyInt_FromSsize_t PyLong_FromSsize_t #define PyInt_AsLong PyLong_AsLong #define PyInt_AS_LONG PyLong_AS_LONG #define PyInt_AsSsize_t PyLong_AsSsize_t #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask #endif #if PY_MAJOR_VERSION >= 3 #define PyBoolObject PyLongObject #endif #if PY_VERSION_HEX < 0x03020000 typedef long Py_hash_t; #define __Pyx_PyInt_FromHash_t PyInt_FromLong #define __Pyx_PyInt_AsHash_t PyInt_AsLong #else #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t #endif #if (PY_MAJOR_VERSION < 3) || (PY_VERSION_HEX >= 0x03010300) #define __Pyx_PySequence_GetSlice(obj, a, b) PySequence_GetSlice(obj, a, b) #define __Pyx_PySequence_SetSlice(obj, a, b, value) PySequence_SetSlice(obj, a, b, value) #define __Pyx_PySequence_DelSlice(obj, a, b) PySequence_DelSlice(obj, a, b) #else #define __Pyx_PySequence_GetSlice(obj, a, b) (unlikely(!(obj)) ? \ (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), (PyObject*)0) : \ (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_GetSlice(obj, a, b)) : \ (PyErr_Format(PyExc_TypeError, "'%.200s' object is unsliceable", (obj)->ob_type->tp_name), (PyObject*)0))) #define __Pyx_PySequence_SetSlice(obj, a, b, value) (unlikely(!(obj)) ? \ (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \ (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_SetSlice(obj, a, b, value)) : \ (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice assignment", (obj)->ob_type->tp_name), -1))) #define __Pyx_PySequence_DelSlice(obj, a, b) (unlikely(!(obj)) ? \ (PyErr_SetString(PyExc_SystemError, "null argument to internal routine"), -1) : \ (likely((obj)->ob_type->tp_as_mapping) ? (PySequence_DelSlice(obj, a, b)) : \ (PyErr_Format(PyExc_TypeError, "'%.200s' object doesn't support slice deletion", (obj)->ob_type->tp_name), -1))) #endif #if PY_MAJOR_VERSION >= 3 #define PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) #endif #if PY_VERSION_HEX < 0x02050000 #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) #else #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) #endif #if PY_VERSION_HEX < 0x02050000 #define __Pyx_NAMESTR(n) ((char *)(n)) #define __Pyx_DOCSTR(n) ((char *)(n)) #else #define __Pyx_NAMESTR(n) (n) #define __Pyx_DOCSTR(n) (n) #endif #ifndef CYTHON_INLINE #if defined(__GNUC__) #define CYTHON_INLINE __inline__ #elif defined(_MSC_VER) #define CYTHON_INLINE __inline #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define CYTHON_INLINE inline #else #define CYTHON_INLINE #endif #endif #ifndef CYTHON_RESTRICT #if defined(__GNUC__) #define CYTHON_RESTRICT __restrict__ #elif defined(_MSC_VER) #define CYTHON_RESTRICT __restrict #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define CYTHON_RESTRICT restrict #else #define CYTHON_RESTRICT #endif #endif #ifdef NAN #define __PYX_NAN() ((float) NAN) #else static CYTHON_INLINE float __PYX_NAN() { /* Initialize NaN. The sign is irrelevant, an exponent with all bits 1 and a nonzero mantissa means NaN. If the first bit in the mantissa is 1, it is a quiet NaN. */ float value; memset(&value, 0xFF, sizeof(value)); return value; } #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) #else #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif #ifndef __PYX_EXTERN_C #ifdef __cplusplus #define __PYX_EXTERN_C extern "C" #else #define __PYX_EXTERN_C extern #endif #endif #if defined(WIN32) || defined(MS_WINDOWS) #define _USE_MATH_DEFINES #endif #include #define __PYX_HAVE__attic__hashindex #define __PYX_HAVE_API__attic__hashindex #include "_hashindex.c" #ifdef _OPENMP #include #endif /* _OPENMP */ #ifdef PYREX_WITHOUT_ASSERTIONS #define CYTHON_WITHOUT_ASSERTIONS #endif #ifndef CYTHON_UNUSED # if defined(__GNUC__) # if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) # define CYTHON_UNUSED __attribute__ ((__unused__)) # else # define CYTHON_UNUSED # endif # elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) # define CYTHON_UNUSED __attribute__ ((__unused__)) # else # define CYTHON_UNUSED # endif #endif typedef struct {PyObject **p; char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/ #define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 #define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 #define __PYX_DEFAULT_STRING_ENCODING "" #define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString #define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); #define __Pyx_PyBytes_FromString PyBytes_FromString #define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char*); #if PY_MAJOR_VERSION < 3 #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize #else #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize #endif #define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromUString(s) __Pyx_PyObject_FromString((char*)s) #define __Pyx_PyBytes_FromUString(s) __Pyx_PyBytes_FromString((char*)s) #define __Pyx_PyStr_FromUString(s) __Pyx_PyStr_FromString((char*)s) #define __Pyx_PyUnicode_FromUString(s) __Pyx_PyUnicode_FromString((char*)s) #if PY_MAJOR_VERSION < 3 static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { const Py_UNICODE *u_end = u; while (*u_end++) ; return u_end - u - 1; } #else #define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen #endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode #define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None) #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); #if CYTHON_COMPILING_IN_CPYTHON #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) #else #define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) #endif #define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; static int __Pyx_init_sys_getdefaultencoding_params() { PyObject* sys = NULL; PyObject* default_encoding = NULL; PyObject* ascii_chars_u = NULL; PyObject* ascii_chars_b = NULL; sys = PyImport_ImportModule("sys"); if (sys == NULL) goto bad; default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); if (default_encoding == NULL) goto bad; if (strcmp(PyBytes_AsString(default_encoding), "ascii") == 0) { __Pyx_sys_getdefaultencoding_not_ascii = 0; } else { const char* default_encoding_c = PyBytes_AS_STRING(default_encoding); char ascii_chars[128]; int c; for (c = 0; c < 128; c++) { ascii_chars[c] = c; } __Pyx_sys_getdefaultencoding_not_ascii = 1; ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); if (ascii_chars_u == NULL) goto bad; ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); if (ascii_chars_b == NULL || strncmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { PyErr_Format( PyExc_ValueError, "This module compiled with c_string_encoding=ascii, but default encoding '%s' is not a superset of ascii.", default_encoding_c); goto bad; } } Py_XDECREF(sys); Py_XDECREF(default_encoding); Py_XDECREF(ascii_chars_u); Py_XDECREF(ascii_chars_b); return 0; bad: Py_XDECREF(sys); Py_XDECREF(default_encoding); Py_XDECREF(ascii_chars_u); Py_XDECREF(ascii_chars_b); return -1; } #endif #if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) #else #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) #if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT static char* __PYX_DEFAULT_STRING_ENCODING; static int __Pyx_init_sys_getdefaultencoding_params() { PyObject* sys = NULL; PyObject* default_encoding = NULL; char* default_encoding_c; sys = PyImport_ImportModule("sys"); if (sys == NULL) goto bad; default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); if (default_encoding == NULL) goto bad; default_encoding_c = PyBytes_AS_STRING(default_encoding); __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); Py_DECREF(sys); Py_DECREF(default_encoding); return 0; bad: Py_XDECREF(sys); Py_XDECREF(default_encoding); return -1; } #endif #endif #ifdef __GNUC__ /* Test for GCC > 2.95 */ #if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #else /* __GNUC__ > 2 ... */ #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ > 2 ... */ #else /* __GNUC__ */ #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ */ static PyObject *__pyx_m; static PyObject *__pyx_d; static PyObject *__pyx_b; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; static int __pyx_lineno; static int __pyx_clineno = 0; static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char *__pyx_f[] = { "hashindex.pyx", }; /*--- Type declarations ---*/ struct __pyx_obj_5attic_9hashindex_IndexBase; struct __pyx_obj_5attic_9hashindex_NSIndex; struct __pyx_obj_5attic_9hashindex_NSKeyIterator; struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator; struct __pyx_obj_5attic_9hashindex_ChunkIndex; /* "attic/hashindex.pyx":25 * _NoDefault = object() * * cdef class IndexBase: # <<<<<<<<<<<<<< * cdef HashIndex *index * key_size = 32 */ struct __pyx_obj_5attic_9hashindex_IndexBase { PyObject_HEAD HashIndex *index; }; /* "attic/hashindex.pyx":84 * * * cdef class NSIndex(IndexBase): # <<<<<<<<<<<<<< * * value_size = 8 */ struct __pyx_obj_5attic_9hashindex_NSIndex { struct __pyx_obj_5attic_9hashindex_IndexBase __pyx_base; }; /* "attic/hashindex.pyx":114 * * * cdef class NSKeyIterator: # <<<<<<<<<<<<<< * cdef HashIndex *index * cdef char *key */ struct __pyx_obj_5attic_9hashindex_NSKeyIterator { PyObject_HEAD HashIndex *index; char *key; }; /* "attic/hashindex.pyx":163 * * * cdef class ChunkKeyIterator: # <<<<<<<<<<<<<< * cdef HashIndex *index * cdef char *key */ struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator { PyObject_HEAD HashIndex *index; char *key; }; /* "attic/hashindex.pyx":132 * * * cdef class ChunkIndex(IndexBase): # <<<<<<<<<<<<<< * * value_size = 12 */ struct __pyx_obj_5attic_9hashindex_ChunkIndex { struct __pyx_obj_5attic_9hashindex_IndexBase __pyx_base; }; #ifndef CYTHON_REFNANNY #define CYTHON_REFNANNY 0 #endif #if CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); void (*DECREF)(void*, PyObject*, int); void (*GOTREF)(void*, PyObject*, int); void (*GIVEREF)(void*, PyObject*, int); void* (*SetupContext)(const char*, int, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); /*proto*/ #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; #ifdef WITH_THREAD #define __Pyx_RefNannySetupContext(name, acquire_gil) \ if (acquire_gil) { \ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ PyGILState_Release(__pyx_gilstate_save); \ } else { \ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil) \ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) #endif #define __Pyx_RefNannyFinishContext() \ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) #define __Pyx_GOTREF(r) #define __Pyx_GIVEREF(r) #define __Pyx_XINCREF(r) Py_XINCREF(r) #define __Pyx_XDECREF(r) Py_XDECREF(r) #define __Pyx_XGOTREF(r) #define __Pyx_XGIVEREF(r) #endif /* CYTHON_REFNANNY */ #define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) #define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { PyTypeObject* tp = Py_TYPE(obj); if (likely(tp->tp_getattro)) return tp->tp_getattro(obj, attr_name); #if PY_MAJOR_VERSION < 3 if (likely(tp->tp_getattr)) return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); #endif return PyObject_GetAttr(obj, attr_name); } #else #define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) #endif static PyObject *__Pyx_GetBuiltinName(PyObject *name); /*proto*/ static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); /*proto*/ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ const char* function_name); /*proto*/ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); /*proto*/ static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); /*proto*/ static CYTHON_INLINE int __Pyx_PySequence_Contains(PyObject* item, PyObject* seq, int eq) { int result = PySequence_Contains(seq, item); return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); } static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ #define __Pyx_GetItemInt(o, i, size, to_py_func, is_list, wraparound, boundscheck) \ (((size) <= sizeof(Py_ssize_t)) ? \ __Pyx_GetItemInt_Fast(o, i, is_list, wraparound, boundscheck) : \ __Pyx_GetItemInt_Generic(o, to_py_func(i))) #define __Pyx_GetItemInt_List(o, i, size, to_py_func, is_list, wraparound, boundscheck) \ (((size) <= sizeof(Py_ssize_t)) ? \ __Pyx_GetItemInt_List_Fast(o, i, wraparound, boundscheck) : \ __Pyx_GetItemInt_Generic(o, to_py_func(i))) static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, int wraparound, int boundscheck); #define __Pyx_GetItemInt_Tuple(o, i, size, to_py_func, is_list, wraparound, boundscheck) \ (((size) <= sizeof(Py_ssize_t)) ? \ __Pyx_GetItemInt_Tuple_Fast(o, i, wraparound, boundscheck) : \ __Pyx_GetItemInt_Generic(o, to_py_func(i))) static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, int wraparound, int boundscheck); static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/ static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name); /*proto*/ static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /*proto*/ #include "descrobject.h" static PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/ static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject *); static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject *); static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject *); static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject *); static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject *); static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); static int __Pyx_check_binary_version(void); typedef struct { int code_line; PyCodeObject* code_object; } __Pyx_CodeObjectCacheEntry; struct __Pyx_CodeObjectCache { int count; int max_count; __Pyx_CodeObjectCacheEntry* entries; }; static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename); /*proto*/ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Module declarations from 'attic.hashindex' */ static PyTypeObject *__pyx_ptype_5attic_9hashindex_IndexBase = 0; static PyTypeObject *__pyx_ptype_5attic_9hashindex_NSIndex = 0; static PyTypeObject *__pyx_ptype_5attic_9hashindex_NSKeyIterator = 0; static PyTypeObject *__pyx_ptype_5attic_9hashindex_ChunkKeyIterator = 0; static PyTypeObject *__pyx_ptype_5attic_9hashindex_ChunkIndex = 0; #define __Pyx_MODULE_NAME "attic.hashindex" int __pyx_module_is_main_attic__hashindex = 0; /* Implementation of 'attic.hashindex' */ static PyObject *__pyx_builtin_object; static PyObject *__pyx_builtin_Exception; static PyObject *__pyx_builtin_KeyError; static PyObject *__pyx_builtin_StopIteration; static int __pyx_pf_5attic_9hashindex_9IndexBase___cinit__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_readonly); /* proto */ static void __pyx_pf_5attic_9hashindex_9IndexBase_2__dealloc__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_4create(PyObject *__pyx_v_cls, PyObject *__pyx_v_path); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_6clear(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_8flush(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_10setdefault(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value); /* proto */ static int __pyx_pf_5attic_9hashindex_9IndexBase_12__delitem__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_14get(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_default); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_16pop(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_default); /* proto */ static Py_ssize_t __pyx_pf_5attic_9hashindex_9IndexBase_18__len__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_7NSIndex___getitem__(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, PyObject *__pyx_v_key); /* proto */ static int __pyx_pf_5attic_9hashindex_7NSIndex_2__setitem__(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value); /* proto */ static int __pyx_pf_5attic_9hashindex_7NSIndex_4__contains__(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, PyObject *__pyx_v_key); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_7NSIndex_6iteritems(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_marker, CYTHON_UNUSED PyObject *__pyx_v_limit); /* proto */ static int __pyx_pf_5attic_9hashindex_13NSKeyIterator___cinit__(struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_13NSKeyIterator_2__iter__(struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_13NSKeyIterator_4__next__(struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_10ChunkIndex___getitem__(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, PyObject *__pyx_v_key); /* proto */ static int __pyx_pf_5attic_9hashindex_10ChunkIndex_2__setitem__(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value); /* proto */ static int __pyx_pf_5attic_9hashindex_10ChunkIndex_4__contains__(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, PyObject *__pyx_v_key); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_10ChunkIndex_6iteritems(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_marker, CYTHON_UNUSED PyObject *__pyx_v_limit); /* proto */ static int __pyx_pf_5attic_9hashindex_16ChunkKeyIterator___cinit__(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_16ChunkKeyIterator_2__iter__(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5attic_9hashindex_16ChunkKeyIterator_4__next__(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_self); /* proto */ static PyObject *__pyx_tp_new_5attic_9hashindex_IndexBase(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5attic_9hashindex_NSIndex(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5attic_9hashindex_NSKeyIterator(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5attic_9hashindex_ChunkKeyIterator(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5attic_9hashindex_ChunkIndex(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static char __pyx_k_2[] = "Failed to open %s"; static char __pyx_k_3[] = "hashindex_close failed"; static char __pyx_k_5[] = "Failed to create %s"; static char __pyx_k_6[] = "hashindex_clear failed"; static char __pyx_k_8[] = "hashindex_flush failed"; static char __pyx_k_10[] = "hashindex_delete failed"; static char __pyx_k_13[] = "hashindex_set failed"; static char __pyx_k__os[] = "os"; static char __pyx_k__key[] = "key"; static char __pyx_k__path[] = "path"; static char __pyx_k__limit[] = "limit"; static char __pyx_k__value[] = "value"; static char __pyx_k__create[] = "create"; static char __pyx_k__marker[] = "marker"; static char __pyx_k__object[] = "object"; static char __pyx_k__default[] = "default"; static char __pyx_k__KeyError[] = "KeyError"; static char __pyx_k____main__[] = "__main__"; static char __pyx_k____test__[] = "__test__"; static char __pyx_k__fsencode[] = "fsencode"; static char __pyx_k__key_size[] = "key_size"; static char __pyx_k__readonly[] = "readonly"; static char __pyx_k__Exception[] = "Exception"; static char __pyx_k___NoDefault[] = "_NoDefault"; static char __pyx_k____import__[] = "__import__"; static char __pyx_k__value_size[] = "value_size"; static char __pyx_k__StopIteration[] = "StopIteration"; static PyObject *__pyx_kp_s_10; static PyObject *__pyx_kp_s_13; static PyObject *__pyx_kp_s_2; static PyObject *__pyx_kp_s_3; static PyObject *__pyx_kp_s_5; static PyObject *__pyx_kp_s_6; static PyObject *__pyx_kp_s_8; static PyObject *__pyx_n_s__Exception; static PyObject *__pyx_n_s__KeyError; static PyObject *__pyx_n_s__StopIteration; static PyObject *__pyx_n_s___NoDefault; static PyObject *__pyx_n_s____import__; static PyObject *__pyx_n_s____main__; static PyObject *__pyx_n_s____test__; static PyObject *__pyx_n_s__create; static PyObject *__pyx_n_s__default; static PyObject *__pyx_n_s__fsencode; static PyObject *__pyx_n_s__key; static PyObject *__pyx_n_s__key_size; static PyObject *__pyx_n_s__limit; static PyObject *__pyx_n_s__marker; static PyObject *__pyx_n_s__object; static PyObject *__pyx_n_s__os; static PyObject *__pyx_n_s__path; static PyObject *__pyx_n_s__readonly; static PyObject *__pyx_n_s__value; static PyObject *__pyx_n_s__value_size; static PyObject *__pyx_int_0; static PyObject *__pyx_int_8; static PyObject *__pyx_int_12; static PyObject *__pyx_int_32; static PyObject *__pyx_k_1; static PyObject *__pyx_k_12; static PyObject *__pyx_k_tuple_4; static PyObject *__pyx_k_tuple_7; static PyObject *__pyx_k_tuple_9; static PyObject *__pyx_k_tuple_11; static PyObject *__pyx_k_tuple_14; static PyObject *__pyx_k_tuple_15; /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_9IndexBase_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5attic_9hashindex_9IndexBase_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_path = 0; PyObject *__pyx_v_readonly = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__path,&__pyx_n_s__readonly,0}; PyObject* values[2] = {0,0}; values[1] = __pyx_k_1; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__path)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__readonly); if (value) { values[1] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_path = values[0]; __pyx_v_readonly = values[1]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.hashindex.IndexBase.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase___cinit__(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self), __pyx_v_path, __pyx_v_readonly); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":29 * key_size = 32 * * def __cinit__(self, path, readonly=False): # <<<<<<<<<<<<<< * self.index = hashindex_open(os.fsencode(path), readonly) * if not self.index: */ static int __pyx_pf_5attic_9hashindex_9IndexBase___cinit__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_readonly) { int __pyx_r; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; char *__pyx_t_4; int __pyx_t_5; int __pyx_t_6; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); /* "attic/hashindex.pyx":30 * * def __cinit__(self, path, readonly=False): * self.index = hashindex_open(os.fsencode(path), readonly) # <<<<<<<<<<<<<< * if not self.index: * raise Exception('Failed to open %s' % path) */ __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s__fsencode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_path); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_path); __Pyx_GIVEREF(__pyx_v_path); __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_t_3); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_5 = __Pyx_PyInt_AsInt(__pyx_v_readonly); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_self->index = hashindex_open(__pyx_t_4, __pyx_t_5); /* "attic/hashindex.pyx":31 * def __cinit__(self, path, readonly=False): * self.index = hashindex_open(os.fsencode(path), readonly) * if not self.index: # <<<<<<<<<<<<<< * raise Exception('Failed to open %s' % path) * */ __pyx_t_6 = ((!(__pyx_v_self->index != 0)) != 0); if (__pyx_t_6) { /* "attic/hashindex.pyx":32 * self.index = hashindex_open(os.fsencode(path), readonly) * if not self.index: * raise Exception('Failed to open %s' % path) # <<<<<<<<<<<<<< * * def __dealloc__(self): */ __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_2), __pyx_v_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_3)); __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_t_3)); __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; __pyx_t_3 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("attic.hashindex.IndexBase.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static void __pyx_pw_5attic_9hashindex_9IndexBase_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ static void __pyx_pw_5attic_9hashindex_9IndexBase_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); __pyx_pf_5attic_9hashindex_9IndexBase_2__dealloc__(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); } /* "attic/hashindex.pyx":34 * raise Exception('Failed to open %s' % path) * * def __dealloc__(self): # <<<<<<<<<<<<<< * if self.index: * if not hashindex_close(self.index): */ static void __pyx_pf_5attic_9hashindex_9IndexBase_2__dealloc__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self) { __Pyx_RefNannyDeclarations int __pyx_t_1; PyObject *__pyx_t_2 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__dealloc__", 0); /* "attic/hashindex.pyx":35 * * def __dealloc__(self): * if self.index: # <<<<<<<<<<<<<< * if not hashindex_close(self.index): * raise Exception('hashindex_close failed') */ __pyx_t_1 = (__pyx_v_self->index != 0); if (__pyx_t_1) { /* "attic/hashindex.pyx":36 * def __dealloc__(self): * if self.index: * if not hashindex_close(self.index): # <<<<<<<<<<<<<< * raise Exception('hashindex_close failed') * */ __pyx_t_1 = ((!(hashindex_close(__pyx_v_self->index) != 0)) != 0); if (__pyx_t_1) { /* "attic/hashindex.pyx":37 * if self.index: * if not hashindex_close(self.index): * raise Exception('hashindex_close failed') # <<<<<<<<<<<<<< * * @classmethod */ __pyx_t_2 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_k_tuple_4), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L4; } __pyx_L4:; goto __pyx_L3; } __pyx_L3:; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.hashindex.IndexBase.__dealloc__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_L0:; __Pyx_RefNannyFinishContext(); } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_5create(PyObject *__pyx_v_cls, PyObject *__pyx_v_path); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_5create(PyObject *__pyx_v_cls, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("create (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_4create(((PyObject*)__pyx_v_cls), ((PyObject *)__pyx_v_path)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":40 * * @classmethod * def create(cls, path): # <<<<<<<<<<<<<< * index = hashindex_create(os.fsencode(path), 0, cls.key_size, cls.value_size) * if not index: */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_4create(PyObject *__pyx_v_cls, PyObject *__pyx_v_path) { HashIndex *__pyx_v_index; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; char *__pyx_t_4; int __pyx_t_5; int __pyx_t_6; int __pyx_t_7; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("create", 0); /* "attic/hashindex.pyx":41 * @classmethod * def create(cls, path): * index = hashindex_create(os.fsencode(path), 0, cls.key_size, cls.value_size) # <<<<<<<<<<<<<< * if not index: * raise Exception('Failed to create %s' % path) */ __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s__fsencode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_path); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_path); __Pyx_GIVEREF(__pyx_v_path); __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_t_3); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_cls), __pyx_n_s__key_size); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_t_5 = __Pyx_PyInt_AsInt(__pyx_t_3); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_cls), __pyx_n_s__value_size); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_t_6 = __Pyx_PyInt_AsInt(__pyx_t_3); if (unlikely((__pyx_t_6 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_v_index = hashindex_create(__pyx_t_4, 0, __pyx_t_5, __pyx_t_6); /* "attic/hashindex.pyx":42 * def create(cls, path): * index = hashindex_create(os.fsencode(path), 0, cls.key_size, cls.value_size) * if not index: # <<<<<<<<<<<<<< * raise Exception('Failed to create %s' % path) * hashindex_close(index) */ __pyx_t_7 = ((!(__pyx_v_index != 0)) != 0); if (__pyx_t_7) { /* "attic/hashindex.pyx":43 * index = hashindex_create(os.fsencode(path), 0, cls.key_size, cls.value_size) * if not index: * raise Exception('Failed to create %s' % path) # <<<<<<<<<<<<<< * hashindex_close(index) * return cls(path) */ __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_5), __pyx_v_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_3)); __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_t_3)); __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; __pyx_t_3 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; /* "attic/hashindex.pyx":44 * if not index: * raise Exception('Failed to create %s' % path) * hashindex_close(index) # <<<<<<<<<<<<<< * return cls(path) * */ hashindex_close(__pyx_v_index); /* "attic/hashindex.pyx":45 * raise Exception('Failed to create %s' % path) * hashindex_close(index) * return cls(path) # <<<<<<<<<<<<<< * * def clear(self): */ __Pyx_XDECREF(__pyx_r); __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_v_path); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_path); __Pyx_GIVEREF(__pyx_v_path); __pyx_t_1 = PyObject_Call(((PyObject *)__pyx_v_cls), ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("attic.hashindex.IndexBase.create", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_7clear(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_7clear(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("clear (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_6clear(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":47 * return cls(path) * * def clear(self): # <<<<<<<<<<<<<< * if not hashindex_clear(self.index): * raise Exception('hashindex_clear failed') */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_6clear(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; PyObject *__pyx_t_2 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("clear", 0); /* "attic/hashindex.pyx":48 * * def clear(self): * if not hashindex_clear(self.index): # <<<<<<<<<<<<<< * raise Exception('hashindex_clear failed') * */ __pyx_t_1 = ((!(hashindex_clear(__pyx_v_self->index) != 0)) != 0); if (__pyx_t_1) { /* "attic/hashindex.pyx":49 * def clear(self): * if not hashindex_clear(self.index): * raise Exception('hashindex_clear failed') # <<<<<<<<<<<<<< * * def flush(self): */ __pyx_t_2 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_k_tuple_7), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.hashindex.IndexBase.clear", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_9flush(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_9flush(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("flush (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_8flush(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":51 * raise Exception('hashindex_clear failed') * * def flush(self): # <<<<<<<<<<<<<< * if not hashindex_flush(self.index): * raise Exception('hashindex_flush failed') */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_8flush(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; PyObject *__pyx_t_2 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("flush", 0); /* "attic/hashindex.pyx":52 * * def flush(self): * if not hashindex_flush(self.index): # <<<<<<<<<<<<<< * raise Exception('hashindex_flush failed') * */ __pyx_t_1 = ((!(hashindex_flush(__pyx_v_self->index) != 0)) != 0); if (__pyx_t_1) { /* "attic/hashindex.pyx":53 * def flush(self): * if not hashindex_flush(self.index): * raise Exception('hashindex_flush failed') # <<<<<<<<<<<<<< * * def setdefault(self, key, value): */ __pyx_t_2 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_k_tuple_9), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.hashindex.IndexBase.flush", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_11setdefault(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_11setdefault(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_key = 0; PyObject *__pyx_v_value = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("setdefault (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__key,&__pyx_n_s__value,0}; PyObject* values[2] = {0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__key)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__value)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("setdefault", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "setdefault") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; } else { values[0] = PyTuple_GET_ITEM(__pyx_args, 0); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); } __pyx_v_key = values[0]; __pyx_v_value = values[1]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("setdefault", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.hashindex.IndexBase.setdefault", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_10setdefault(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self), __pyx_v_key, __pyx_v_value); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":55 * raise Exception('hashindex_flush failed') * * def setdefault(self, key, value): # <<<<<<<<<<<<<< * if not key in self: * self[key] = value */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_10setdefault(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; int __pyx_t_2; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("setdefault", 0); /* "attic/hashindex.pyx":56 * * def setdefault(self, key, value): * if not key in self: # <<<<<<<<<<<<<< * self[key] = value * */ __pyx_t_1 = (__Pyx_PySequence_Contains(__pyx_v_key, ((PyObject *)__pyx_v_self), Py_EQ)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_2 = ((!(__pyx_t_1 != 0)) != 0); if (__pyx_t_2) { /* "attic/hashindex.pyx":57 * def setdefault(self, key, value): * if not key in self: * self[key] = value # <<<<<<<<<<<<<< * * def __delitem__(self, key): */ if (PyObject_SetItem(((PyObject *)__pyx_v_self), __pyx_v_key, __pyx_v_value) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_AddTraceback("attic.hashindex.IndexBase.setdefault", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_9IndexBase_13__delitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key); /*proto*/ static int __pyx_pw_5attic_9hashindex_9IndexBase_13__delitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__delitem__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_12__delitem__(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self), ((PyObject *)__pyx_v_key)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":59 * self[key] = value * * def __delitem__(self, key): # <<<<<<<<<<<<<< * assert len(key) == 32 * if not hashindex_delete(self.index, key): */ static int __pyx_pf_5attic_9hashindex_9IndexBase_12__delitem__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key) { int __pyx_r; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; char *__pyx_t_2; int __pyx_t_3; PyObject *__pyx_t_4 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__delitem__", 0); /* "attic/hashindex.pyx":60 * * def __delitem__(self, key): * assert len(key) == 32 # <<<<<<<<<<<<<< * if not hashindex_delete(self.index, key): * raise Exception('hashindex_delete failed') */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":61 * def __delitem__(self, key): * assert len(key) == 32 * if not hashindex_delete(self.index, key): # <<<<<<<<<<<<<< * raise Exception('hashindex_delete failed') * */ __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_3 = ((!(hashindex_delete(__pyx_v_self->index, ((char *)__pyx_t_2)) != 0)) != 0); if (__pyx_t_3) { /* "attic/hashindex.pyx":62 * assert len(key) == 32 * if not hashindex_delete(self.index, key): * raise Exception('hashindex_delete failed') # <<<<<<<<<<<<<< * * def get(self, key, default=None): */ __pyx_t_4 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_k_tuple_11), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("attic.hashindex.IndexBase.__delitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_15get(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_15get(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_key = 0; PyObject *__pyx_v_default = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("get (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__key,&__pyx_n_s__default,0}; PyObject* values[2] = {0,0}; /* "attic/hashindex.pyx":64 * raise Exception('hashindex_delete failed') * * def get(self, key, default=None): # <<<<<<<<<<<<<< * try: * return self[key] */ values[1] = ((PyObject *)Py_None); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__key)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__default); if (value) { values[1] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "get") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_key = values[0]; __pyx_v_default = values[1]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("get", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.hashindex.IndexBase.get", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_14get(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self), __pyx_v_key, __pyx_v_default); __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_14get(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_default) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; int __pyx_t_5; PyObject *__pyx_t_6 = NULL; PyObject *__pyx_t_7 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("get", 0); /* "attic/hashindex.pyx":65 * * def get(self, key, default=None): * try: # <<<<<<<<<<<<<< * return self[key] * except KeyError: */ { __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); __Pyx_XGOTREF(__pyx_t_1); __Pyx_XGOTREF(__pyx_t_2); __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { /* "attic/hashindex.pyx":66 * def get(self, key, default=None): * try: * return self[key] # <<<<<<<<<<<<<< * except KeyError: * return default */ __Pyx_XDECREF(__pyx_r); __pyx_t_4 = PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_key); if (!__pyx_t_4) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L7_try_return; } __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; goto __pyx_L10_try_end; __pyx_L7_try_return:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); goto __pyx_L0; __pyx_L3_error:; __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; /* "attic/hashindex.pyx":67 * try: * return self[key] * except KeyError: # <<<<<<<<<<<<<< * return default * */ __pyx_t_5 = PyErr_ExceptionMatches(__pyx_builtin_KeyError); if (__pyx_t_5) { __Pyx_AddTraceback("attic.hashindex.IndexBase.get", __pyx_clineno, __pyx_lineno, __pyx_filename); if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_6, &__pyx_t_7) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); /* "attic/hashindex.pyx":68 * return self[key] * except KeyError: * return default # <<<<<<<<<<<<<< * * def pop(self, key, default=_NoDefault): */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_default); __pyx_r = __pyx_v_default; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L6_except_return; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L4_exception_handled; } __pyx_L5_except_error:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); goto __pyx_L1_error; __pyx_L6_except_return:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); goto __pyx_L0; __pyx_L4_exception_handled:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); __pyx_L10_try_end:; } __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_AddTraceback("attic.hashindex.IndexBase.get", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_17pop(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_9IndexBase_17pop(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_key = 0; PyObject *__pyx_v_default = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("pop (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__key,&__pyx_n_s__default,0}; PyObject* values[2] = {0,0}; values[1] = __pyx_k_12; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__key)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__default); if (value) { values[1] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "pop") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_key = values[0]; __pyx_v_default = values[1]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("pop", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.hashindex.IndexBase.pop", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_16pop(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self), __pyx_v_key, __pyx_v_default); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":70 * return default * * def pop(self, key, default=_NoDefault): # <<<<<<<<<<<<<< * try: * value = self[key] */ static PyObject *__pyx_pf_5attic_9hashindex_9IndexBase_16pop(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_default) { PyObject *__pyx_v_value = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; int __pyx_t_5; PyObject *__pyx_t_6 = NULL; PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; PyObject *__pyx_t_9 = NULL; int __pyx_t_10; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("pop", 0); /* "attic/hashindex.pyx":71 * * def pop(self, key, default=_NoDefault): * try: # <<<<<<<<<<<<<< * value = self[key] * del self[key] */ { __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); __Pyx_XGOTREF(__pyx_t_1); __Pyx_XGOTREF(__pyx_t_2); __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { /* "attic/hashindex.pyx":72 * def pop(self, key, default=_NoDefault): * try: * value = self[key] # <<<<<<<<<<<<<< * del self[key] * return value */ __pyx_t_4 = PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_key); if (!__pyx_t_4) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_v_value = __pyx_t_4; __pyx_t_4 = 0; /* "attic/hashindex.pyx":73 * try: * value = self[key] * del self[key] # <<<<<<<<<<<<<< * return value * except KeyError: */ if (PyObject_DelItem(((PyObject *)__pyx_v_self), __pyx_v_key) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L3_error;} /* "attic/hashindex.pyx":74 * value = self[key] * del self[key] * return value # <<<<<<<<<<<<<< * except KeyError: * if default != _NoDefault: */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_value); __pyx_r = __pyx_v_value; goto __pyx_L7_try_return; } __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; goto __pyx_L10_try_end; __pyx_L7_try_return:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); goto __pyx_L0; __pyx_L3_error:; __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; /* "attic/hashindex.pyx":75 * del self[key] * return value * except KeyError: # <<<<<<<<<<<<<< * if default != _NoDefault: * return default */ __pyx_t_5 = PyErr_ExceptionMatches(__pyx_builtin_KeyError); if (__pyx_t_5) { __Pyx_AddTraceback("attic.hashindex.IndexBase.pop", __pyx_clineno, __pyx_lineno, __pyx_filename); if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_6, &__pyx_t_7) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_GOTREF(__pyx_t_6); __Pyx_GOTREF(__pyx_t_7); /* "attic/hashindex.pyx":76 * return value * except KeyError: * if default != _NoDefault: # <<<<<<<<<<<<<< * return default * raise */ __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s___NoDefault); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_GOTREF(__pyx_t_8); __pyx_t_9 = PyObject_RichCompare(__pyx_v_default, __pyx_t_8, Py_NE); __Pyx_XGOTREF(__pyx_t_9); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_10 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; if (__pyx_t_10) { /* "attic/hashindex.pyx":77 * except KeyError: * if default != _NoDefault: * return default # <<<<<<<<<<<<<< * raise * */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_default); __pyx_r = __pyx_v_default; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L6_except_return; goto __pyx_L13; } __pyx_L13:; /* "attic/hashindex.pyx":78 * if default != _NoDefault: * return default * raise # <<<<<<<<<<<<<< * * def __len__(self): */ __Pyx_GIVEREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_7); __Pyx_ErrRestore(__pyx_t_4, __pyx_t_6, __pyx_t_7); __pyx_t_4 = 0; __pyx_t_6 = 0; __pyx_t_7 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L4_exception_handled; } __pyx_L5_except_error:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); goto __pyx_L1_error; __pyx_L6_except_return:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); goto __pyx_L0; __pyx_L4_exception_handled:; __Pyx_XGIVEREF(__pyx_t_1); __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); __pyx_L10_try_end:; } __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); __Pyx_XDECREF(__pyx_t_9); __Pyx_AddTraceback("attic.hashindex.IndexBase.pop", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF(__pyx_v_value); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static Py_ssize_t __pyx_pw_5attic_9hashindex_9IndexBase_19__len__(PyObject *__pyx_v_self); /*proto*/ static Py_ssize_t __pyx_pw_5attic_9hashindex_9IndexBase_19__len__(PyObject *__pyx_v_self) { Py_ssize_t __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_9IndexBase_18__len__(((struct __pyx_obj_5attic_9hashindex_IndexBase *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":80 * raise * * def __len__(self): # <<<<<<<<<<<<<< * return hashindex_get_size(self.index) * */ static Py_ssize_t __pyx_pf_5attic_9hashindex_9IndexBase_18__len__(struct __pyx_obj_5attic_9hashindex_IndexBase *__pyx_v_self) { Py_ssize_t __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); /* "attic/hashindex.pyx":81 * * def __len__(self): * return hashindex_get_size(self.index) # <<<<<<<<<<<<<< * * */ __pyx_r = hashindex_get_size(__pyx_v_self->index); goto __pyx_L0; __pyx_r = 0; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_7NSIndex_1__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_7NSIndex_1__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_7NSIndex___getitem__(((struct __pyx_obj_5attic_9hashindex_NSIndex *)__pyx_v_self), ((PyObject *)__pyx_v_key)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":88 * value_size = 8 * * def __getitem__(self, key): # <<<<<<<<<<<<<< * assert len(key) == 32 * data = hashindex_get(self.index, key) */ static PyObject *__pyx_pf_5attic_9hashindex_7NSIndex___getitem__(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, PyObject *__pyx_v_key) { int *__pyx_v_data; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; char *__pyx_t_2; int __pyx_t_3; PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; PyObject *__pyx_t_6 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); /* "attic/hashindex.pyx":89 * * def __getitem__(self, key): * assert len(key) == 32 # <<<<<<<<<<<<<< * data = hashindex_get(self.index, key) * if not data: */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":90 * def __getitem__(self, key): * assert len(key) == 32 * data = hashindex_get(self.index, key) # <<<<<<<<<<<<<< * if not data: * raise KeyError */ __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_data = ((int *)hashindex_get(__pyx_v_self->__pyx_base.index, ((char *)__pyx_t_2))); /* "attic/hashindex.pyx":91 * assert len(key) == 32 * data = hashindex_get(self.index, key) * if not data: # <<<<<<<<<<<<<< * raise KeyError * return _le32toh(data[0]), _le32toh(data[1]) */ __pyx_t_3 = ((!(__pyx_v_data != 0)) != 0); if (__pyx_t_3) { /* "attic/hashindex.pyx":92 * data = hashindex_get(self.index, key) * if not data: * raise KeyError # <<<<<<<<<<<<<< * return _le32toh(data[0]), _le32toh(data[1]) * */ __Pyx_Raise(__pyx_builtin_KeyError, 0, 0, 0); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; /* "attic/hashindex.pyx":93 * if not data: * raise KeyError * return _le32toh(data[0]), _le32toh(data[1]) # <<<<<<<<<<<<<< * * def __setitem__(self, key, value): */ __Pyx_XDECREF(__pyx_r); __pyx_t_4 = PyInt_FromLong(_le32toh((__pyx_v_data[0]))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = PyInt_FromLong(_le32toh((__pyx_v_data[1]))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); __pyx_t_4 = 0; __pyx_t_5 = 0; __pyx_r = ((PyObject *)__pyx_t_6); __pyx_t_6 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); __Pyx_XDECREF(__pyx_t_6); __Pyx_AddTraceback("attic.hashindex.NSIndex.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_7NSIndex_3__setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value); /*proto*/ static int __pyx_pw_5attic_9hashindex_7NSIndex_3__setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_7NSIndex_2__setitem__(((struct __pyx_obj_5attic_9hashindex_NSIndex *)__pyx_v_self), ((PyObject *)__pyx_v_key), ((PyObject *)__pyx_v_value)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":95 * return _le32toh(data[0]), _le32toh(data[1]) * * def __setitem__(self, key, value): # <<<<<<<<<<<<<< * assert len(key) == 32 * cdef int[2] data */ static int __pyx_pf_5attic_9hashindex_7NSIndex_2__setitem__(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value) { int __pyx_v_data[2]; int __pyx_r; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; PyObject *__pyx_t_2 = NULL; int __pyx_t_3; char *__pyx_t_4; int __pyx_t_5; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__setitem__", 0); /* "attic/hashindex.pyx":96 * * def __setitem__(self, key, value): * assert len(key) == 32 # <<<<<<<<<<<<<< * cdef int[2] data * data[0] = _htole32(value[0]) */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":98 * assert len(key) == 32 * cdef int[2] data * data[0] = _htole32(value[0]) # <<<<<<<<<<<<<< * data[1] = _htole32(value[1]) * if not hashindex_set(self.index, key, data): */ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_value, 0, sizeof(long), PyInt_FromLong, 0, 0, 1); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_AsInt(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; (__pyx_v_data[0]) = _htole32(__pyx_t_3); /* "attic/hashindex.pyx":99 * cdef int[2] data * data[0] = _htole32(value[0]) * data[1] = _htole32(value[1]) # <<<<<<<<<<<<<< * if not hashindex_set(self.index, key, data): * raise Exception('hashindex_set failed') */ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_value, 1, sizeof(long), PyInt_FromLong, 0, 0, 1); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_AsInt(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; (__pyx_v_data[1]) = _htole32(__pyx_t_3); /* "attic/hashindex.pyx":100 * data[0] = _htole32(value[0]) * data[1] = _htole32(value[1]) * if not hashindex_set(self.index, key, data): # <<<<<<<<<<<<<< * raise Exception('hashindex_set failed') * */ __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_5 = ((!(hashindex_set(__pyx_v_self->__pyx_base.index, ((char *)__pyx_t_4), __pyx_v_data) != 0)) != 0); if (__pyx_t_5) { /* "attic/hashindex.pyx":101 * data[1] = _htole32(value[1]) * if not hashindex_set(self.index, key, data): * raise Exception('hashindex_set failed') # <<<<<<<<<<<<<< * * def __contains__(self, key): */ __pyx_t_2 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_k_tuple_14), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.hashindex.NSIndex.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_7NSIndex_5__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_key); /*proto*/ static int __pyx_pw_5attic_9hashindex_7NSIndex_5__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_key) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__contains__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_7NSIndex_4__contains__(((struct __pyx_obj_5attic_9hashindex_NSIndex *)__pyx_v_self), ((PyObject *)__pyx_v_key)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":103 * raise Exception('hashindex_set failed') * * def __contains__(self, key): # <<<<<<<<<<<<<< * assert len(key) == 32 * data = hashindex_get(self.index, key) */ static int __pyx_pf_5attic_9hashindex_7NSIndex_4__contains__(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, PyObject *__pyx_v_key) { int *__pyx_v_data; int __pyx_r; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; char *__pyx_t_2; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__contains__", 0); /* "attic/hashindex.pyx":104 * * def __contains__(self, key): * assert len(key) == 32 # <<<<<<<<<<<<<< * data = hashindex_get(self.index, key) * return data != NULL */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":105 * def __contains__(self, key): * assert len(key) == 32 * data = hashindex_get(self.index, key) # <<<<<<<<<<<<<< * return data != NULL * */ __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_data = ((int *)hashindex_get(__pyx_v_self->__pyx_base.index, ((char *)__pyx_t_2))); /* "attic/hashindex.pyx":106 * assert len(key) == 32 * data = hashindex_get(self.index, key) * return data != NULL # <<<<<<<<<<<<<< * * def iteritems(self, marker=None, limit=0): */ __pyx_r = (__pyx_v_data != NULL); goto __pyx_L0; __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_AddTraceback("attic.hashindex.NSIndex.__contains__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_7NSIndex_7iteritems(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_7NSIndex_7iteritems(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { CYTHON_UNUSED PyObject *__pyx_v_marker = 0; CYTHON_UNUSED PyObject *__pyx_v_limit = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("iteritems (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__marker,&__pyx_n_s__limit,0}; PyObject* values[2] = {0,0}; /* "attic/hashindex.pyx":108 * return data != NULL * * def iteritems(self, marker=None, limit=0): # <<<<<<<<<<<<<< * iter = NSKeyIterator() * iter.index = self.index */ values[0] = ((PyObject *)Py_None); values[1] = ((PyObject *)__pyx_int_0); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__marker); if (value) { values[0] = value; kw_args--; } } case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__limit); if (value) { values[1] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "iteritems") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_marker = values[0]; __pyx_v_limit = values[1]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("iteritems", 0, 0, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.hashindex.NSIndex.iteritems", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_9hashindex_7NSIndex_6iteritems(((struct __pyx_obj_5attic_9hashindex_NSIndex *)__pyx_v_self), __pyx_v_marker, __pyx_v_limit); __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_5attic_9hashindex_7NSIndex_6iteritems(struct __pyx_obj_5attic_9hashindex_NSIndex *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_marker, CYTHON_UNUSED PyObject *__pyx_v_limit) { struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_iter = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; HashIndex *__pyx_t_2; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("iteritems", 0); /* "attic/hashindex.pyx":109 * * def iteritems(self, marker=None, limit=0): * iter = NSKeyIterator() # <<<<<<<<<<<<<< * iter.index = self.index * return iter */ __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5attic_9hashindex_NSKeyIterator)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_v_iter = ((struct __pyx_obj_5attic_9hashindex_NSKeyIterator *)__pyx_t_1); __pyx_t_1 = 0; /* "attic/hashindex.pyx":110 * def iteritems(self, marker=None, limit=0): * iter = NSKeyIterator() * iter.index = self.index # <<<<<<<<<<<<<< * return iter * */ __pyx_t_2 = __pyx_v_self->__pyx_base.index; __pyx_v_iter->index = __pyx_t_2; /* "attic/hashindex.pyx":111 * iter = NSKeyIterator() * iter.index = self.index * return iter # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_iter)); __pyx_r = ((PyObject *)__pyx_v_iter); goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("attic.hashindex.NSIndex.iteritems", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF((PyObject *)__pyx_v_iter); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_13NSKeyIterator_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5attic_9hashindex_13NSKeyIterator_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__cinit__", 0))) return -1; __pyx_r = __pyx_pf_5attic_9hashindex_13NSKeyIterator___cinit__(((struct __pyx_obj_5attic_9hashindex_NSKeyIterator *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":118 * cdef char *key * * def __cinit__(self): # <<<<<<<<<<<<<< * self.key = NULL * */ static int __pyx_pf_5attic_9hashindex_13NSKeyIterator___cinit__(struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_self) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__", 0); /* "attic/hashindex.pyx":119 * * def __cinit__(self): * self.key = NULL # <<<<<<<<<<<<<< * * def __iter__(self): */ __pyx_v_self->key = NULL; __pyx_r = 0; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_13NSKeyIterator_3__iter__(PyObject *__pyx_v_self); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_13NSKeyIterator_3__iter__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_13NSKeyIterator_2__iter__(((struct __pyx_obj_5attic_9hashindex_NSKeyIterator *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":121 * self.key = NULL * * def __iter__(self): # <<<<<<<<<<<<<< * return self * */ static PyObject *__pyx_pf_5attic_9hashindex_13NSKeyIterator_2__iter__(struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iter__", 0); /* "attic/hashindex.pyx":122 * * def __iter__(self): * return self # <<<<<<<<<<<<<< * * def __next__(self): */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_13NSKeyIterator_5__next__(PyObject *__pyx_v_self); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_13NSKeyIterator_5__next__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__next__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_13NSKeyIterator_4__next__(((struct __pyx_obj_5attic_9hashindex_NSKeyIterator *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":124 * return self * * def __next__(self): # <<<<<<<<<<<<<< * self.key = hashindex_next_key(self.index, self.key) * if not self.key: */ static PyObject *__pyx_pf_5attic_9hashindex_13NSKeyIterator_4__next__(struct __pyx_obj_5attic_9hashindex_NSKeyIterator *__pyx_v_self) { int *__pyx_v_value; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__next__", 0); /* "attic/hashindex.pyx":125 * * def __next__(self): * self.key = hashindex_next_key(self.index, self.key) # <<<<<<<<<<<<<< * if not self.key: * raise StopIteration */ __pyx_v_self->key = ((char *)hashindex_next_key(__pyx_v_self->index, ((char *)__pyx_v_self->key))); /* "attic/hashindex.pyx":126 * def __next__(self): * self.key = hashindex_next_key(self.index, self.key) * if not self.key: # <<<<<<<<<<<<<< * raise StopIteration * cdef int *value = (self.key + 32) */ __pyx_t_1 = ((!(__pyx_v_self->key != 0)) != 0); if (__pyx_t_1) { /* "attic/hashindex.pyx":127 * self.key = hashindex_next_key(self.index, self.key) * if not self.key: * raise StopIteration # <<<<<<<<<<<<<< * cdef int *value = (self.key + 32) * return self.key[:32], (_le32toh(value[0]), _le32toh(value[1])) */ __Pyx_Raise(__pyx_builtin_StopIteration, 0, 0, 0); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; /* "attic/hashindex.pyx":128 * if not self.key: * raise StopIteration * cdef int *value = (self.key + 32) # <<<<<<<<<<<<<< * return self.key[:32], (_le32toh(value[0]), _le32toh(value[1])) * */ __pyx_v_value = ((int *)(__pyx_v_self->key + 32)); /* "attic/hashindex.pyx":129 * raise StopIteration * cdef int *value = (self.key + 32) * return self.key[:32], (_le32toh(value[0]), _le32toh(value[1])) # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); __pyx_t_2 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_self->key + 0, 32 - 0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __pyx_t_3 = PyInt_FromLong(_le32toh((__pyx_v_value[0]))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyInt_FromLong(_le32toh((__pyx_v_value[1]))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_3 = 0; __pyx_t_4 = 0; __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_4, 0, ((PyObject *)__pyx_t_2)); __Pyx_GIVEREF(((PyObject *)__pyx_t_2)); PyTuple_SET_ITEM(__pyx_t_4, 1, ((PyObject *)__pyx_t_5)); __Pyx_GIVEREF(((PyObject *)__pyx_t_5)); __pyx_t_2 = 0; __pyx_t_5 = 0; __pyx_r = ((PyObject *)__pyx_t_4); __pyx_t_4 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); __Pyx_AddTraceback("attic.hashindex.NSKeyIterator.__next__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_10ChunkIndex_1__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_10ChunkIndex_1__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_10ChunkIndex___getitem__(((struct __pyx_obj_5attic_9hashindex_ChunkIndex *)__pyx_v_self), ((PyObject *)__pyx_v_key)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":136 * value_size = 12 * * def __getitem__(self, key): # <<<<<<<<<<<<<< * assert len(key) == 32 * data = hashindex_get(self.index, key) */ static PyObject *__pyx_pf_5attic_9hashindex_10ChunkIndex___getitem__(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, PyObject *__pyx_v_key) { int *__pyx_v_data; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; char *__pyx_t_2; int __pyx_t_3; PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; PyObject *__pyx_t_6 = NULL; PyObject *__pyx_t_7 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); /* "attic/hashindex.pyx":137 * * def __getitem__(self, key): * assert len(key) == 32 # <<<<<<<<<<<<<< * data = hashindex_get(self.index, key) * if not data: */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":138 * def __getitem__(self, key): * assert len(key) == 32 * data = hashindex_get(self.index, key) # <<<<<<<<<<<<<< * if not data: * raise KeyError */ __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_data = ((int *)hashindex_get(__pyx_v_self->__pyx_base.index, ((char *)__pyx_t_2))); /* "attic/hashindex.pyx":139 * assert len(key) == 32 * data = hashindex_get(self.index, key) * if not data: # <<<<<<<<<<<<<< * raise KeyError * return _le32toh(data[0]), _le32toh(data[1]), _le32toh(data[2]) */ __pyx_t_3 = ((!(__pyx_v_data != 0)) != 0); if (__pyx_t_3) { /* "attic/hashindex.pyx":140 * data = hashindex_get(self.index, key) * if not data: * raise KeyError # <<<<<<<<<<<<<< * return _le32toh(data[0]), _le32toh(data[1]), _le32toh(data[2]) * */ __Pyx_Raise(__pyx_builtin_KeyError, 0, 0, 0); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; /* "attic/hashindex.pyx":141 * if not data: * raise KeyError * return _le32toh(data[0]), _le32toh(data[1]), _le32toh(data[2]) # <<<<<<<<<<<<<< * * def __setitem__(self, key, value): */ __Pyx_XDECREF(__pyx_r); __pyx_t_4 = PyInt_FromLong(_le32toh((__pyx_v_data[0]))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = PyInt_FromLong(_le32toh((__pyx_v_data[1]))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = PyInt_FromLong(_le32toh((__pyx_v_data[2]))); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_6); __Pyx_GIVEREF(__pyx_t_6); __pyx_t_4 = 0; __pyx_t_5 = 0; __pyx_t_6 = 0; __pyx_r = ((PyObject *)__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_AddTraceback("attic.hashindex.ChunkIndex.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_10ChunkIndex_3__setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value); /*proto*/ static int __pyx_pw_5attic_9hashindex_10ChunkIndex_3__setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_10ChunkIndex_2__setitem__(((struct __pyx_obj_5attic_9hashindex_ChunkIndex *)__pyx_v_self), ((PyObject *)__pyx_v_key), ((PyObject *)__pyx_v_value)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":143 * return _le32toh(data[0]), _le32toh(data[1]), _le32toh(data[2]) * * def __setitem__(self, key, value): # <<<<<<<<<<<<<< * assert len(key) == 32 * cdef int[3] data */ static int __pyx_pf_5attic_9hashindex_10ChunkIndex_2__setitem__(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, PyObject *__pyx_v_key, PyObject *__pyx_v_value) { int __pyx_v_data[3]; int __pyx_r; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; PyObject *__pyx_t_2 = NULL; int __pyx_t_3; char *__pyx_t_4; int __pyx_t_5; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__setitem__", 0); /* "attic/hashindex.pyx":144 * * def __setitem__(self, key, value): * assert len(key) == 32 # <<<<<<<<<<<<<< * cdef int[3] data * data[0] = _htole32(value[0]) */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":146 * assert len(key) == 32 * cdef int[3] data * data[0] = _htole32(value[0]) # <<<<<<<<<<<<<< * data[1] = _htole32(value[1]) * data[2] = _htole32(value[2]) */ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_value, 0, sizeof(long), PyInt_FromLong, 0, 0, 1); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_AsInt(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; (__pyx_v_data[0]) = _htole32(__pyx_t_3); /* "attic/hashindex.pyx":147 * cdef int[3] data * data[0] = _htole32(value[0]) * data[1] = _htole32(value[1]) # <<<<<<<<<<<<<< * data[2] = _htole32(value[2]) * if not hashindex_set(self.index, key, data): */ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_value, 1, sizeof(long), PyInt_FromLong, 0, 0, 1); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_AsInt(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; (__pyx_v_data[1]) = _htole32(__pyx_t_3); /* "attic/hashindex.pyx":148 * data[0] = _htole32(value[0]) * data[1] = _htole32(value[1]) * data[2] = _htole32(value[2]) # <<<<<<<<<<<<<< * if not hashindex_set(self.index, key, data): * raise Exception('hashindex_set failed') */ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_value, 2, sizeof(long), PyInt_FromLong, 0, 0, 1); if (!__pyx_t_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyInt_AsInt(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; (__pyx_v_data[2]) = _htole32(__pyx_t_3); /* "attic/hashindex.pyx":149 * data[1] = _htole32(value[1]) * data[2] = _htole32(value[2]) * if not hashindex_set(self.index, key, data): # <<<<<<<<<<<<<< * raise Exception('hashindex_set failed') * */ __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 149; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_t_5 = ((!(hashindex_set(__pyx_v_self->__pyx_base.index, ((char *)__pyx_t_4), __pyx_v_data) != 0)) != 0); if (__pyx_t_5) { /* "attic/hashindex.pyx":150 * data[2] = _htole32(value[2]) * if not hashindex_set(self.index, key, data): * raise Exception('hashindex_set failed') # <<<<<<<<<<<<<< * * def __contains__(self, key): */ __pyx_t_2 = PyObject_Call(__pyx_builtin_Exception, ((PyObject *)__pyx_k_tuple_15), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("attic.hashindex.ChunkIndex.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_10ChunkIndex_5__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_key); /*proto*/ static int __pyx_pw_5attic_9hashindex_10ChunkIndex_5__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_key) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__contains__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_10ChunkIndex_4__contains__(((struct __pyx_obj_5attic_9hashindex_ChunkIndex *)__pyx_v_self), ((PyObject *)__pyx_v_key)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":152 * raise Exception('hashindex_set failed') * * def __contains__(self, key): # <<<<<<<<<<<<<< * assert len(key) == 32 * data = hashindex_get(self.index, key) */ static int __pyx_pf_5attic_9hashindex_10ChunkIndex_4__contains__(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, PyObject *__pyx_v_key) { int *__pyx_v_data; int __pyx_r; __Pyx_RefNannyDeclarations Py_ssize_t __pyx_t_1; char *__pyx_t_2; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__contains__", 0); /* "attic/hashindex.pyx":153 * * def __contains__(self, key): * assert len(key) == 32 # <<<<<<<<<<<<<< * data = hashindex_get(self.index, key) * return data != NULL */ #ifndef CYTHON_WITHOUT_ASSERTIONS __pyx_t_1 = PyObject_Length(__pyx_v_key); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!((__pyx_t_1 == 32) != 0))) { PyErr_SetNone(PyExc_AssertionError); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #endif /* "attic/hashindex.pyx":154 * def __contains__(self, key): * assert len(key) == 32 * data = hashindex_get(self.index, key) # <<<<<<<<<<<<<< * return data != NULL * */ __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_key); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 154; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_data = ((int *)hashindex_get(__pyx_v_self->__pyx_base.index, ((char *)__pyx_t_2))); /* "attic/hashindex.pyx":155 * assert len(key) == 32 * data = hashindex_get(self.index, key) * return data != NULL # <<<<<<<<<<<<<< * * def iteritems(self, marker=None, limit=0): */ __pyx_r = (__pyx_v_data != NULL); goto __pyx_L0; __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_AddTraceback("attic.hashindex.ChunkIndex.__contains__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_10ChunkIndex_7iteritems(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_10ChunkIndex_7iteritems(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { CYTHON_UNUSED PyObject *__pyx_v_marker = 0; CYTHON_UNUSED PyObject *__pyx_v_limit = 0; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("iteritems (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__marker,&__pyx_n_s__limit,0}; PyObject* values[2] = {0,0}; /* "attic/hashindex.pyx":157 * return data != NULL * * def iteritems(self, marker=None, limit=0): # <<<<<<<<<<<<<< * iter = ChunkKeyIterator() * iter.index = self.index */ values[0] = ((PyObject *)Py_None); values[1] = ((PyObject *)__pyx_int_0); if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__marker); if (value) { values[0] = value; kw_args--; } } case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__limit); if (value) { values[1] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "iteritems") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_marker = values[0]; __pyx_v_limit = values[1]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("iteritems", 0, 0, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("attic.hashindex.ChunkIndex.iteritems", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_r = __pyx_pf_5attic_9hashindex_10ChunkIndex_6iteritems(((struct __pyx_obj_5attic_9hashindex_ChunkIndex *)__pyx_v_self), __pyx_v_marker, __pyx_v_limit); __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_5attic_9hashindex_10ChunkIndex_6iteritems(struct __pyx_obj_5attic_9hashindex_ChunkIndex *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_marker, CYTHON_UNUSED PyObject *__pyx_v_limit) { struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_iter = NULL; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; HashIndex *__pyx_t_2; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("iteritems", 0); /* "attic/hashindex.pyx":158 * * def iteritems(self, marker=None, limit=0): * iter = ChunkKeyIterator() # <<<<<<<<<<<<<< * iter.index = self.index * return iter */ __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5attic_9hashindex_ChunkKeyIterator)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_v_iter = ((struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *)__pyx_t_1); __pyx_t_1 = 0; /* "attic/hashindex.pyx":159 * def iteritems(self, marker=None, limit=0): * iter = ChunkKeyIterator() * iter.index = self.index # <<<<<<<<<<<<<< * return iter * */ __pyx_t_2 = __pyx_v_self->__pyx_base.index; __pyx_v_iter->index = __pyx_t_2; /* "attic/hashindex.pyx":160 * iter = ChunkKeyIterator() * iter.index = self.index * return iter # <<<<<<<<<<<<<< * * */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_iter)); __pyx_r = ((PyObject *)__pyx_v_iter); goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("attic.hashindex.ChunkIndex.iteritems", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF((PyObject *)__pyx_v_iter); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static int __pyx_pw_5attic_9hashindex_16ChunkKeyIterator_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5attic_9hashindex_16ChunkKeyIterator_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__cinit__", 0))) return -1; __pyx_r = __pyx_pf_5attic_9hashindex_16ChunkKeyIterator___cinit__(((struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":167 * cdef char *key * * def __cinit__(self): # <<<<<<<<<<<<<< * self.key = NULL * */ static int __pyx_pf_5attic_9hashindex_16ChunkKeyIterator___cinit__(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_self) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__", 0); /* "attic/hashindex.pyx":168 * * def __cinit__(self): * self.key = NULL # <<<<<<<<<<<<<< * * def __iter__(self): */ __pyx_v_self->key = NULL; __pyx_r = 0; __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_16ChunkKeyIterator_3__iter__(PyObject *__pyx_v_self); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_16ChunkKeyIterator_3__iter__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_16ChunkKeyIterator_2__iter__(((struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":170 * self.key = NULL * * def __iter__(self): # <<<<<<<<<<<<<< * return self * */ static PyObject *__pyx_pf_5attic_9hashindex_16ChunkKeyIterator_2__iter__(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iter__", 0); /* "attic/hashindex.pyx":171 * * def __iter__(self): * return self # <<<<<<<<<<<<<< * * def __next__(self): */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* Python wrapper */ static PyObject *__pyx_pw_5attic_9hashindex_16ChunkKeyIterator_5__next__(PyObject *__pyx_v_self); /*proto*/ static PyObject *__pyx_pw_5attic_9hashindex_16ChunkKeyIterator_5__next__(PyObject *__pyx_v_self) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__next__ (wrapper)", 0); __pyx_r = __pyx_pf_5attic_9hashindex_16ChunkKeyIterator_4__next__(((struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "attic/hashindex.pyx":173 * return self * * def __next__(self): # <<<<<<<<<<<<<< * self.key = hashindex_next_key(self.index, self.key) * if not self.key: */ static PyObject *__pyx_pf_5attic_9hashindex_16ChunkKeyIterator_4__next__(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator *__pyx_v_self) { int *__pyx_v_value; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; PyObject *__pyx_t_6 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__next__", 0); /* "attic/hashindex.pyx":174 * * def __next__(self): * self.key = hashindex_next_key(self.index, self.key) # <<<<<<<<<<<<<< * if not self.key: * raise StopIteration */ __pyx_v_self->key = ((char *)hashindex_next_key(__pyx_v_self->index, ((char *)__pyx_v_self->key))); /* "attic/hashindex.pyx":175 * def __next__(self): * self.key = hashindex_next_key(self.index, self.key) * if not self.key: # <<<<<<<<<<<<<< * raise StopIteration * cdef int *value = (self.key + 32) */ __pyx_t_1 = ((!(__pyx_v_self->key != 0)) != 0); if (__pyx_t_1) { /* "attic/hashindex.pyx":176 * self.key = hashindex_next_key(self.index, self.key) * if not self.key: * raise StopIteration # <<<<<<<<<<<<<< * cdef int *value = (self.key + 32) * return self.key[:32], (_le32toh(value[0]), _le32toh(value[1]), _le32toh(value[2])) */ __Pyx_Raise(__pyx_builtin_StopIteration, 0, 0, 0); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L3; } __pyx_L3:; /* "attic/hashindex.pyx":177 * if not self.key: * raise StopIteration * cdef int *value = (self.key + 32) # <<<<<<<<<<<<<< * return self.key[:32], (_le32toh(value[0]), _le32toh(value[1]), _le32toh(value[2])) */ __pyx_v_value = ((int *)(__pyx_v_self->key + 32)); /* "attic/hashindex.pyx":178 * raise StopIteration * cdef int *value = (self.key + 32) * return self.key[:32], (_le32toh(value[0]), _le32toh(value[1]), _le32toh(value[2])) # <<<<<<<<<<<<<< */ __Pyx_XDECREF(__pyx_r); __pyx_t_2 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_self->key + 0, 32 - 0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __pyx_t_3 = PyInt_FromLong(_le32toh((__pyx_v_value[0]))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyInt_FromLong(_le32toh((__pyx_v_value[1]))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = PyInt_FromLong(_le32toh((__pyx_v_value[2]))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = PyTuple_New(3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 2, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); __pyx_t_3 = 0; __pyx_t_4 = 0; __pyx_t_5 = 0; __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_5, 0, ((PyObject *)__pyx_t_2)); __Pyx_GIVEREF(((PyObject *)__pyx_t_2)); PyTuple_SET_ITEM(__pyx_t_5, 1, ((PyObject *)__pyx_t_6)); __Pyx_GIVEREF(((PyObject *)__pyx_t_6)); __pyx_t_2 = 0; __pyx_t_6 = 0; __pyx_r = ((PyObject *)__pyx_t_5); __pyx_t_5 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); __Pyx_XDECREF(__pyx_t_6); __Pyx_AddTraceback("attic.hashindex.ChunkKeyIterator.__next__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_tp_new_5attic_9hashindex_IndexBase(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o; o = (*t->tp_alloc)(t, 0); if (unlikely(!o)) return 0; if (unlikely(__pyx_pw_5attic_9hashindex_9IndexBase_1__cinit__(o, a, k) < 0)) { Py_DECREF(o); o = 0; } return o; } static void __pyx_tp_dealloc_5attic_9hashindex_IndexBase(PyObject *o) { { PyObject *etype, *eval, *etb; PyErr_Fetch(&etype, &eval, &etb); ++Py_REFCNT(o); __pyx_pw_5attic_9hashindex_9IndexBase_3__dealloc__(o); if (PyErr_Occurred()) PyErr_WriteUnraisable(o); --Py_REFCNT(o); PyErr_Restore(etype, eval, etb); } (*Py_TYPE(o)->tp_free)(o); } static int __pyx_mp_ass_subscript_5attic_9hashindex_IndexBase(PyObject *o, PyObject *i, PyObject *v) { if (v) { PyErr_Format(PyExc_NotImplementedError, "Subscript assignment not supported by %s", Py_TYPE(o)->tp_name); return -1; } else { return __pyx_pw_5attic_9hashindex_9IndexBase_13__delitem__(o, i); } } static PyMethodDef __pyx_methods_5attic_9hashindex_IndexBase[] = { {__Pyx_NAMESTR("create"), (PyCFunction)__pyx_pw_5attic_9hashindex_9IndexBase_5create, METH_O, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("clear"), (PyCFunction)__pyx_pw_5attic_9hashindex_9IndexBase_7clear, METH_NOARGS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pw_5attic_9hashindex_9IndexBase_9flush, METH_NOARGS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("setdefault"), (PyCFunction)__pyx_pw_5attic_9hashindex_9IndexBase_11setdefault, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("get"), (PyCFunction)__pyx_pw_5attic_9hashindex_9IndexBase_15get, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("pop"), (PyCFunction)__pyx_pw_5attic_9hashindex_9IndexBase_17pop, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; static PySequenceMethods __pyx_tp_as_sequence_IndexBase = { __pyx_pw_5attic_9hashindex_9IndexBase_19__len__, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ 0, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ 0, /*sq_ass_slice*/ 0, /*sq_contains*/ 0, /*sq_inplace_concat*/ 0, /*sq_inplace_repeat*/ }; static PyMappingMethods __pyx_tp_as_mapping_IndexBase = { __pyx_pw_5attic_9hashindex_9IndexBase_19__len__, /*mp_length*/ 0, /*mp_subscript*/ __pyx_mp_ass_subscript_5attic_9hashindex_IndexBase, /*mp_ass_subscript*/ }; static PyTypeObject __pyx_type_5attic_9hashindex_IndexBase = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("attic.hashindex.IndexBase"), /*tp_name*/ sizeof(struct __pyx_obj_5attic_9hashindex_IndexBase), /*tp_basicsize*/ 0, /*tp_itemsize*/ __pyx_tp_dealloc_5attic_9hashindex_IndexBase, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else 0, /*reserved*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ &__pyx_tp_as_sequence_IndexBase, /*tp_as_sequence*/ &__pyx_tp_as_mapping_IndexBase, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ __pyx_methods_5attic_9hashindex_IndexBase, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ __pyx_tp_new_5attic_9hashindex_IndexBase, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ 0, /*tp_del*/ #if PY_VERSION_HEX >= 0x02060000 0, /*tp_version_tag*/ #endif }; static PyObject *__pyx_tp_new_5attic_9hashindex_NSIndex(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = __pyx_tp_new_5attic_9hashindex_IndexBase(t, a, k); if (unlikely(!o)) return 0; return o; } static PyObject *__pyx_sq_item_5attic_9hashindex_NSIndex(PyObject *o, Py_ssize_t i) { PyObject *r; PyObject *x = PyInt_FromSsize_t(i); if(!x) return 0; r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); Py_DECREF(x); return r; } static int __pyx_mp_ass_subscript_5attic_9hashindex_NSIndex(PyObject *o, PyObject *i, PyObject *v) { if (v) { return __pyx_pw_5attic_9hashindex_7NSIndex_3__setitem__(o, i, v); } else { if (__pyx_ptype_5attic_9hashindex_IndexBase->tp_as_mapping && __pyx_ptype_5attic_9hashindex_IndexBase->tp_as_mapping->mp_ass_subscript) return __pyx_ptype_5attic_9hashindex_IndexBase->tp_as_mapping->mp_ass_subscript(o, i, v); PyErr_Format(PyExc_NotImplementedError, "Subscript deletion not supported by %s", Py_TYPE(o)->tp_name); return -1; } } static PyMethodDef __pyx_methods_5attic_9hashindex_NSIndex[] = { {__Pyx_NAMESTR("iteritems"), (PyCFunction)__pyx_pw_5attic_9hashindex_7NSIndex_7iteritems, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; static PySequenceMethods __pyx_tp_as_sequence_NSIndex = { #if CYTHON_COMPILING_IN_PYPY __pyx_pw_5attic_9hashindex_9IndexBase_19__len__, /*sq_length*/ #else 0, /*sq_length*/ #endif 0, /*sq_concat*/ 0, /*sq_repeat*/ __pyx_sq_item_5attic_9hashindex_NSIndex, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ 0, /*sq_ass_slice*/ __pyx_pw_5attic_9hashindex_7NSIndex_5__contains__, /*sq_contains*/ 0, /*sq_inplace_concat*/ 0, /*sq_inplace_repeat*/ }; static PyMappingMethods __pyx_tp_as_mapping_NSIndex = { #if CYTHON_COMPILING_IN_PYPY __pyx_pw_5attic_9hashindex_9IndexBase_19__len__, /*mp_length*/ #else 0, /*mp_length*/ #endif __pyx_pw_5attic_9hashindex_7NSIndex_1__getitem__, /*mp_subscript*/ __pyx_mp_ass_subscript_5attic_9hashindex_NSIndex, /*mp_ass_subscript*/ }; static PyTypeObject __pyx_type_5attic_9hashindex_NSIndex = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("attic.hashindex.NSIndex"), /*tp_name*/ sizeof(struct __pyx_obj_5attic_9hashindex_NSIndex), /*tp_basicsize*/ 0, /*tp_itemsize*/ __pyx_tp_dealloc_5attic_9hashindex_IndexBase, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else 0, /*reserved*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ &__pyx_tp_as_sequence_NSIndex, /*tp_as_sequence*/ &__pyx_tp_as_mapping_NSIndex, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ __pyx_methods_5attic_9hashindex_NSIndex, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ __pyx_tp_new_5attic_9hashindex_NSIndex, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ 0, /*tp_del*/ #if PY_VERSION_HEX >= 0x02060000 0, /*tp_version_tag*/ #endif }; static PyObject *__pyx_tp_new_5attic_9hashindex_NSKeyIterator(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { PyObject *o; o = (*t->tp_alloc)(t, 0); if (unlikely(!o)) return 0; if (unlikely(__pyx_pw_5attic_9hashindex_13NSKeyIterator_1__cinit__(o, __pyx_empty_tuple, NULL) < 0)) { Py_DECREF(o); o = 0; } return o; } static void __pyx_tp_dealloc_5attic_9hashindex_NSKeyIterator(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } static PyMethodDef __pyx_methods_5attic_9hashindex_NSKeyIterator[] = { {__Pyx_NAMESTR("__next__"), (PyCFunction)__pyx_pw_5attic_9hashindex_13NSKeyIterator_5__next__, METH_NOARGS|METH_COEXIST, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; static PyTypeObject __pyx_type_5attic_9hashindex_NSKeyIterator = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("attic.hashindex.NSKeyIterator"), /*tp_name*/ sizeof(struct __pyx_obj_5attic_9hashindex_NSKeyIterator), /*tp_basicsize*/ 0, /*tp_itemsize*/ __pyx_tp_dealloc_5attic_9hashindex_NSKeyIterator, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else 0, /*reserved*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ __pyx_pw_5attic_9hashindex_13NSKeyIterator_3__iter__, /*tp_iter*/ __pyx_pw_5attic_9hashindex_13NSKeyIterator_5__next__, /*tp_iternext*/ __pyx_methods_5attic_9hashindex_NSKeyIterator, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ __pyx_tp_new_5attic_9hashindex_NSKeyIterator, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ 0, /*tp_del*/ #if PY_VERSION_HEX >= 0x02060000 0, /*tp_version_tag*/ #endif }; static PyObject *__pyx_tp_new_5attic_9hashindex_ChunkKeyIterator(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { PyObject *o; o = (*t->tp_alloc)(t, 0); if (unlikely(!o)) return 0; if (unlikely(__pyx_pw_5attic_9hashindex_16ChunkKeyIterator_1__cinit__(o, __pyx_empty_tuple, NULL) < 0)) { Py_DECREF(o); o = 0; } return o; } static void __pyx_tp_dealloc_5attic_9hashindex_ChunkKeyIterator(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } static PyMethodDef __pyx_methods_5attic_9hashindex_ChunkKeyIterator[] = { {__Pyx_NAMESTR("__next__"), (PyCFunction)__pyx_pw_5attic_9hashindex_16ChunkKeyIterator_5__next__, METH_NOARGS|METH_COEXIST, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; static PyTypeObject __pyx_type_5attic_9hashindex_ChunkKeyIterator = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("attic.hashindex.ChunkKeyIterator"), /*tp_name*/ sizeof(struct __pyx_obj_5attic_9hashindex_ChunkKeyIterator), /*tp_basicsize*/ 0, /*tp_itemsize*/ __pyx_tp_dealloc_5attic_9hashindex_ChunkKeyIterator, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else 0, /*reserved*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ __pyx_pw_5attic_9hashindex_16ChunkKeyIterator_3__iter__, /*tp_iter*/ __pyx_pw_5attic_9hashindex_16ChunkKeyIterator_5__next__, /*tp_iternext*/ __pyx_methods_5attic_9hashindex_ChunkKeyIterator, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ __pyx_tp_new_5attic_9hashindex_ChunkKeyIterator, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ 0, /*tp_del*/ #if PY_VERSION_HEX >= 0x02060000 0, /*tp_version_tag*/ #endif }; static PyObject *__pyx_tp_new_5attic_9hashindex_ChunkIndex(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = __pyx_tp_new_5attic_9hashindex_IndexBase(t, a, k); if (unlikely(!o)) return 0; return o; } static PyObject *__pyx_sq_item_5attic_9hashindex_ChunkIndex(PyObject *o, Py_ssize_t i) { PyObject *r; PyObject *x = PyInt_FromSsize_t(i); if(!x) return 0; r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); Py_DECREF(x); return r; } static int __pyx_mp_ass_subscript_5attic_9hashindex_ChunkIndex(PyObject *o, PyObject *i, PyObject *v) { if (v) { return __pyx_pw_5attic_9hashindex_10ChunkIndex_3__setitem__(o, i, v); } else { if (__pyx_ptype_5attic_9hashindex_IndexBase->tp_as_mapping && __pyx_ptype_5attic_9hashindex_IndexBase->tp_as_mapping->mp_ass_subscript) return __pyx_ptype_5attic_9hashindex_IndexBase->tp_as_mapping->mp_ass_subscript(o, i, v); PyErr_Format(PyExc_NotImplementedError, "Subscript deletion not supported by %s", Py_TYPE(o)->tp_name); return -1; } } static PyMethodDef __pyx_methods_5attic_9hashindex_ChunkIndex[] = { {__Pyx_NAMESTR("iteritems"), (PyCFunction)__pyx_pw_5attic_9hashindex_10ChunkIndex_7iteritems, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; static PySequenceMethods __pyx_tp_as_sequence_ChunkIndex = { #if CYTHON_COMPILING_IN_PYPY __pyx_pw_5attic_9hashindex_9IndexBase_19__len__, /*sq_length*/ #else 0, /*sq_length*/ #endif 0, /*sq_concat*/ 0, /*sq_repeat*/ __pyx_sq_item_5attic_9hashindex_ChunkIndex, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ 0, /*sq_ass_slice*/ __pyx_pw_5attic_9hashindex_10ChunkIndex_5__contains__, /*sq_contains*/ 0, /*sq_inplace_concat*/ 0, /*sq_inplace_repeat*/ }; static PyMappingMethods __pyx_tp_as_mapping_ChunkIndex = { #if CYTHON_COMPILING_IN_PYPY __pyx_pw_5attic_9hashindex_9IndexBase_19__len__, /*mp_length*/ #else 0, /*mp_length*/ #endif __pyx_pw_5attic_9hashindex_10ChunkIndex_1__getitem__, /*mp_subscript*/ __pyx_mp_ass_subscript_5attic_9hashindex_ChunkIndex, /*mp_ass_subscript*/ }; static PyTypeObject __pyx_type_5attic_9hashindex_ChunkIndex = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("attic.hashindex.ChunkIndex"), /*tp_name*/ sizeof(struct __pyx_obj_5attic_9hashindex_ChunkIndex), /*tp_basicsize*/ 0, /*tp_itemsize*/ __pyx_tp_dealloc_5attic_9hashindex_IndexBase, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else 0, /*reserved*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ &__pyx_tp_as_sequence_ChunkIndex, /*tp_as_sequence*/ &__pyx_tp_as_mapping_ChunkIndex, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ 0, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ __pyx_methods_5attic_9hashindex_ChunkIndex, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ 0, /*tp_descr_set*/ 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ __pyx_tp_new_5attic_9hashindex_ChunkIndex, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ 0, /*tp_mro*/ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ 0, /*tp_del*/ #if PY_VERSION_HEX >= 0x02060000 0, /*tp_version_tag*/ #endif }; static PyMethodDef __pyx_methods[] = { {0, 0, 0, 0} }; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef __pyx_moduledef = { #if PY_VERSION_HEX < 0x03020000 { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, #else PyModuleDef_HEAD_INIT, #endif __Pyx_NAMESTR("hashindex"), 0, /* m_doc */ -1, /* m_size */ __pyx_methods /* m_methods */, NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL /* m_free */ }; #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_10, __pyx_k_10, sizeof(__pyx_k_10), 0, 0, 1, 0}, {&__pyx_kp_s_13, __pyx_k_13, sizeof(__pyx_k_13), 0, 0, 1, 0}, {&__pyx_kp_s_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 1, 0}, {&__pyx_kp_s_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 1, 0}, {&__pyx_kp_s_5, __pyx_k_5, sizeof(__pyx_k_5), 0, 0, 1, 0}, {&__pyx_kp_s_6, __pyx_k_6, sizeof(__pyx_k_6), 0, 0, 1, 0}, {&__pyx_kp_s_8, __pyx_k_8, sizeof(__pyx_k_8), 0, 0, 1, 0}, {&__pyx_n_s__Exception, __pyx_k__Exception, sizeof(__pyx_k__Exception), 0, 0, 1, 1}, {&__pyx_n_s__KeyError, __pyx_k__KeyError, sizeof(__pyx_k__KeyError), 0, 0, 1, 1}, {&__pyx_n_s__StopIteration, __pyx_k__StopIteration, sizeof(__pyx_k__StopIteration), 0, 0, 1, 1}, {&__pyx_n_s___NoDefault, __pyx_k___NoDefault, sizeof(__pyx_k___NoDefault), 0, 0, 1, 1}, {&__pyx_n_s____import__, __pyx_k____import__, sizeof(__pyx_k____import__), 0, 0, 1, 1}, {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1}, {&__pyx_n_s____test__, __pyx_k____test__, sizeof(__pyx_k____test__), 0, 0, 1, 1}, {&__pyx_n_s__create, __pyx_k__create, sizeof(__pyx_k__create), 0, 0, 1, 1}, {&__pyx_n_s__default, __pyx_k__default, sizeof(__pyx_k__default), 0, 0, 1, 1}, {&__pyx_n_s__fsencode, __pyx_k__fsencode, sizeof(__pyx_k__fsencode), 0, 0, 1, 1}, {&__pyx_n_s__key, __pyx_k__key, sizeof(__pyx_k__key), 0, 0, 1, 1}, {&__pyx_n_s__key_size, __pyx_k__key_size, sizeof(__pyx_k__key_size), 0, 0, 1, 1}, {&__pyx_n_s__limit, __pyx_k__limit, sizeof(__pyx_k__limit), 0, 0, 1, 1}, {&__pyx_n_s__marker, __pyx_k__marker, sizeof(__pyx_k__marker), 0, 0, 1, 1}, {&__pyx_n_s__object, __pyx_k__object, sizeof(__pyx_k__object), 0, 0, 1, 1}, {&__pyx_n_s__os, __pyx_k__os, sizeof(__pyx_k__os), 0, 0, 1, 1}, {&__pyx_n_s__path, __pyx_k__path, sizeof(__pyx_k__path), 0, 0, 1, 1}, {&__pyx_n_s__readonly, __pyx_k__readonly, sizeof(__pyx_k__readonly), 0, 0, 1, 1}, {&__pyx_n_s__value, __pyx_k__value, sizeof(__pyx_k__value), 0, 0, 1, 1}, {&__pyx_n_s__value_size, __pyx_k__value_size, sizeof(__pyx_k__value_size), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_object = __Pyx_GetBuiltinName(__pyx_n_s__object); if (!__pyx_builtin_object) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_Exception = __Pyx_GetBuiltinName(__pyx_n_s__Exception); if (!__pyx_builtin_Exception) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_KeyError = __Pyx_GetBuiltinName(__pyx_n_s__KeyError); if (!__pyx_builtin_KeyError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_StopIteration = __Pyx_GetBuiltinName(__pyx_n_s__StopIteration); if (!__pyx_builtin_StopIteration) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; } static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); /* "attic/hashindex.pyx":37 * if self.index: * if not hashindex_close(self.index): * raise Exception('hashindex_close failed') # <<<<<<<<<<<<<< * * @classmethod */ __pyx_k_tuple_4 = PyTuple_Pack(1, ((PyObject *)__pyx_kp_s_3)); if (unlikely(!__pyx_k_tuple_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_4); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_4)); /* "attic/hashindex.pyx":49 * def clear(self): * if not hashindex_clear(self.index): * raise Exception('hashindex_clear failed') # <<<<<<<<<<<<<< * * def flush(self): */ __pyx_k_tuple_7 = PyTuple_Pack(1, ((PyObject *)__pyx_kp_s_6)); if (unlikely(!__pyx_k_tuple_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_7); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_7)); /* "attic/hashindex.pyx":53 * def flush(self): * if not hashindex_flush(self.index): * raise Exception('hashindex_flush failed') # <<<<<<<<<<<<<< * * def setdefault(self, key, value): */ __pyx_k_tuple_9 = PyTuple_Pack(1, ((PyObject *)__pyx_kp_s_8)); if (unlikely(!__pyx_k_tuple_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_9); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_9)); /* "attic/hashindex.pyx":62 * assert len(key) == 32 * if not hashindex_delete(self.index, key): * raise Exception('hashindex_delete failed') # <<<<<<<<<<<<<< * * def get(self, key, default=None): */ __pyx_k_tuple_11 = PyTuple_Pack(1, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_k_tuple_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_11); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_11)); /* "attic/hashindex.pyx":101 * data[1] = _htole32(value[1]) * if not hashindex_set(self.index, key, data): * raise Exception('hashindex_set failed') # <<<<<<<<<<<<<< * * def __contains__(self, key): */ __pyx_k_tuple_14 = PyTuple_Pack(1, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_k_tuple_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_14); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_14)); /* "attic/hashindex.pyx":150 * data[2] = _htole32(value[2]) * if not hashindex_set(self.index, key, data): * raise Exception('hashindex_set failed') # <<<<<<<<<<<<<< * * def __contains__(self, key): */ __pyx_k_tuple_15 = PyTuple_Pack(1, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_k_tuple_15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_15); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_15)); __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; __Pyx_RefNannyFinishContext(); return -1; } static int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __pyx_int_8 = PyInt_FromLong(8); if (unlikely(!__pyx_int_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __pyx_int_12 = PyInt_FromLong(12); if (unlikely(!__pyx_int_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; __pyx_int_32 = PyInt_FromLong(32); if (unlikely(!__pyx_int_32)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; return 0; __pyx_L1_error:; return -1; } #if PY_MAJOR_VERSION < 3 PyMODINIT_FUNC inithashindex(void); /*proto*/ PyMODINIT_FUNC inithashindex(void) #else PyMODINIT_FUNC PyInit_hashindex(void); /*proto*/ PyMODINIT_FUNC PyInit_hashindex(void) #endif { PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannyDeclarations #if CYTHON_REFNANNY __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); if (!__Pyx_RefNanny) { PyErr_Clear(); __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); if (!__Pyx_RefNanny) Py_FatalError("failed to import 'refnanny' module"); } #endif __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_hashindex(void)", 0); if ( __Pyx_check_binary_version() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #ifdef __Pyx_CyFunction_USED if (__Pyx_CyFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif #ifdef __Pyx_FusedFunction_USED if (__pyx_FusedFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif #ifdef __Pyx_Generator_USED if (__pyx_Generator_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif /*--- Library function declarations ---*/ /*--- Threads initialization code ---*/ #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS #ifdef WITH_THREAD /* Python build with threading support? */ PyEval_InitThreads(); #endif #endif /*--- Module creation code ---*/ #if PY_MAJOR_VERSION < 3 __pyx_m = Py_InitModule4(__Pyx_NAMESTR("hashindex"), __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); #else __pyx_m = PyModule_Create(&__pyx_moduledef); #endif if (unlikely(!__pyx_m)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} Py_INCREF(__pyx_d); #if PY_MAJOR_VERSION >= 3 { PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (!PyDict_GetItemString(modules, "attic.hashindex")) { if (unlikely(PyDict_SetItemString(modules, "attic.hashindex", __pyx_m) < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } } #endif __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (unlikely(!__pyx_b)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #if CYTHON_COMPILING_IN_PYPY Py_INCREF(__pyx_b); #endif if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; /*--- Initialize various global constants etc. ---*/ if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) if (__Pyx_init_sys_getdefaultencoding_params() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif if (__pyx_module_is_main_attic__hashindex) { if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_n_s____main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} /*--- Constants init code ---*/ if (unlikely(__Pyx_InitCachedConstants() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} /*--- Global init code ---*/ /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ if (PyType_Ready(&__pyx_type_5attic_9hashindex_IndexBase) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "IndexBase", (PyObject *)&__pyx_type_5attic_9hashindex_IndexBase) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5attic_9hashindex_IndexBase = &__pyx_type_5attic_9hashindex_IndexBase; __pyx_type_5attic_9hashindex_NSIndex.tp_base = __pyx_ptype_5attic_9hashindex_IndexBase; if (PyType_Ready(&__pyx_type_5attic_9hashindex_NSIndex) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "NSIndex", (PyObject *)&__pyx_type_5attic_9hashindex_NSIndex) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5attic_9hashindex_NSIndex = &__pyx_type_5attic_9hashindex_NSIndex; if (PyType_Ready(&__pyx_type_5attic_9hashindex_NSKeyIterator) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "NSKeyIterator", (PyObject *)&__pyx_type_5attic_9hashindex_NSKeyIterator) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5attic_9hashindex_NSKeyIterator = &__pyx_type_5attic_9hashindex_NSKeyIterator; if (PyType_Ready(&__pyx_type_5attic_9hashindex_ChunkKeyIterator) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 163; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "ChunkKeyIterator", (PyObject *)&__pyx_type_5attic_9hashindex_ChunkKeyIterator) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 163; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5attic_9hashindex_ChunkKeyIterator = &__pyx_type_5attic_9hashindex_ChunkKeyIterator; __pyx_type_5attic_9hashindex_ChunkIndex.tp_base = __pyx_ptype_5attic_9hashindex_IndexBase; if (PyType_Ready(&__pyx_type_5attic_9hashindex_ChunkIndex) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "ChunkIndex", (PyObject *)&__pyx_type_5attic_9hashindex_ChunkIndex) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5attic_9hashindex_ChunkIndex = &__pyx_type_5attic_9hashindex_ChunkIndex; /*--- Type import code ---*/ /*--- Variable import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ /* "attic/hashindex.pyx":2 * # -*- coding: utf-8 -*- * import os # <<<<<<<<<<<<<< * * */ __pyx_t_1 = __Pyx_Import(((PyObject *)__pyx_n_s__os), 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 2; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s__os, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 2; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "attic/hashindex.pyx":23 * * * _NoDefault = object() # <<<<<<<<<<<<<< * * cdef class IndexBase: */ __pyx_t_1 = PyObject_Call(__pyx_builtin_object, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s___NoDefault, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "attic/hashindex.pyx":27 * cdef class IndexBase: * cdef HashIndex *index * key_size = 32 # <<<<<<<<<<<<<< * * def __cinit__(self, path, readonly=False): */ if (PyDict_SetItem((PyObject *)__pyx_ptype_5attic_9hashindex_IndexBase->tp_dict, __pyx_n_s__key_size, __pyx_int_32) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;} PyType_Modified(__pyx_ptype_5attic_9hashindex_IndexBase); /* "attic/hashindex.pyx":29 * key_size = 32 * * def __cinit__(self, path, readonly=False): # <<<<<<<<<<<<<< * self.index = hashindex_open(os.fsencode(path), readonly) * if not self.index: */ __pyx_t_1 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_k_1 = __pyx_t_1; __Pyx_GIVEREF(__pyx_t_1); __pyx_t_1 = 0; /* "attic/hashindex.pyx":40 * * @classmethod * def create(cls, path): # <<<<<<<<<<<<<< * index = hashindex_create(os.fsencode(path), 0, cls.key_size, cls.value_size) * if not index: */ __pyx_t_1 = __Pyx_GetNameInClass((PyObject *)__pyx_ptype_5attic_9hashindex_IndexBase, __pyx_n_s__create); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (PyDict_SetItem((PyObject *)__pyx_ptype_5attic_9hashindex_IndexBase->tp_dict, __pyx_n_s__create, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5attic_9hashindex_IndexBase); /* "attic/hashindex.pyx":70 * return default * * def pop(self, key, default=_NoDefault): # <<<<<<<<<<<<<< * try: * value = self[key] */ __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s___NoDefault); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_k_12 = __pyx_t_2; __Pyx_GIVEREF(__pyx_t_2); __pyx_t_2 = 0; /* "attic/hashindex.pyx":86 * cdef class NSIndex(IndexBase): * * value_size = 8 # <<<<<<<<<<<<<< * * def __getitem__(self, key): */ if (PyDict_SetItem((PyObject *)__pyx_ptype_5attic_9hashindex_NSIndex->tp_dict, __pyx_n_s__value_size, __pyx_int_8) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} PyType_Modified(__pyx_ptype_5attic_9hashindex_NSIndex); /* "attic/hashindex.pyx":134 * cdef class ChunkIndex(IndexBase): * * value_size = 12 # <<<<<<<<<<<<<< * * def __getitem__(self, key): */ if (PyDict_SetItem((PyObject *)__pyx_ptype_5attic_9hashindex_ChunkIndex->tp_dict, __pyx_n_s__value_size, __pyx_int_12) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} PyType_Modified(__pyx_ptype_5attic_9hashindex_ChunkIndex); /* "attic/hashindex.pyx":1 * # -*- coding: utf-8 -*- # <<<<<<<<<<<<<< * import os * */ __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); if (PyDict_SetItem(__pyx_d, __pyx_n_s____test__, ((PyObject *)__pyx_t_2)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); if (__pyx_m) { __Pyx_AddTraceback("init attic.hashindex", __pyx_clineno, __pyx_lineno, __pyx_filename); Py_DECREF(__pyx_m); __pyx_m = 0; } else if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ImportError, "init attic.hashindex"); } __pyx_L0:; __Pyx_RefNannyFinishContext(); #if PY_MAJOR_VERSION < 3 return; #else return __pyx_m; #endif } /* Runtime support code */ #if CYTHON_REFNANNY static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { PyObject *m = NULL, *p = NULL; void *r = NULL; m = PyImport_ImportModule((char *)modname); if (!m) goto end; p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); if (!p) goto end; r = PyLong_AsVoidPtr(p); end: Py_XDECREF(p); Py_XDECREF(m); return (__Pyx_RefNannyAPIStruct *)r; } #endif /* CYTHON_REFNANNY */ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); if (unlikely(!result)) { PyErr_Format(PyExc_NameError, #if PY_MAJOR_VERSION >= 3 "name '%U' is not defined", name); #else "name '%s' is not defined", PyString_AS_STRING(name)); #endif } return result; } static void __Pyx_RaiseDoubleKeywordsError( const char* func_name, PyObject* kw_name) { PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION >= 3 "%s() got multiple values for keyword argument '%U'", func_name, kw_name); #else "%s() got multiple values for keyword argument '%s'", func_name, PyString_AsString(kw_name)); #endif } static int __Pyx_ParseOptionalKeywords( PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name) { PyObject *key = 0, *value = 0; Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { values[name-argnames] = value; continue; } name = first_kw_arg; #if PY_MAJOR_VERSION < 3 if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { while (*name) { if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) && _PyString_Eq(**name, key)) { values[name-argnames] = value; break; } name++; } if (*name) continue; else { PyObject*** argname = argnames; while (argname != first_kw_arg) { if ((**argname == key) || ( (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) && _PyString_Eq(**argname, key))) { goto arg_passed_twice; } argname++; } } } else #endif if (likely(PyUnicode_Check(key))) { while (*name) { int cmp = (**name == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : #endif PyUnicode_Compare(**name, key); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; break; } name++; } if (*name) continue; else { PyObject*** argname = argnames; while (argname != first_kw_arg) { int cmp = (**argname == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : #endif PyUnicode_Compare(**argname, key); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) goto arg_passed_twice; argname++; } } } else goto invalid_keyword_type; if (kwds2) { if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; } else { goto invalid_keyword; } } return 0; arg_passed_twice: __Pyx_RaiseDoubleKeywordsError(function_name, key); goto bad; invalid_keyword_type: PyErr_Format(PyExc_TypeError, "%s() keywords must be strings", function_name); goto bad; invalid_keyword: PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 "%s() got an unexpected keyword argument '%s'", function_name, PyString_AsString(key)); #else "%s() got an unexpected keyword argument '%U'", function_name, key); #endif bad: return -1; } static void __Pyx_RaiseArgtupleInvalid( const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found) { Py_ssize_t num_expected; const char *more_or_less; if (num_found < num_min) { num_expected = num_min; more_or_less = "at least"; } else { num_expected = num_max; more_or_less = "at most"; } if (exact) { more_or_less = "exactly"; } PyErr_Format(PyExc_TypeError, "%s() takes %s %" CYTHON_FORMAT_SSIZE_T "d positional argument%s (%" CYTHON_FORMAT_SSIZE_T "d given)", func_name, more_or_less, num_expected, (num_expected == 1) ? "" : "s", num_found); } static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { PyObject *result; #if CYTHON_COMPILING_IN_CPYTHON result = PyDict_GetItem(__pyx_d, name); if (result) { Py_INCREF(result); } else { #else result = PyObject_GetItem(__pyx_d, name); if (!result) { PyErr_Clear(); #endif result = __Pyx_GetBuiltinName(name); } return result; } static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { #if CYTHON_COMPILING_IN_CPYTHON PyObject *tmp_type, *tmp_value, *tmp_tb; PyThreadState *tstate = PyThreadState_GET(); tmp_type = tstate->curexc_type; tmp_value = tstate->curexc_value; tmp_tb = tstate->curexc_traceback; tstate->curexc_type = type; tstate->curexc_value = value; tstate->curexc_traceback = tb; Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); #else PyErr_Restore(type, value, tb); #endif } static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { #if CYTHON_COMPILING_IN_CPYTHON PyThreadState *tstate = PyThreadState_GET(); *type = tstate->curexc_type; *value = tstate->curexc_value; *tb = tstate->curexc_traceback; tstate->curexc_type = 0; tstate->curexc_value = 0; tstate->curexc_traceback = 0; #else PyErr_Fetch(type, value, tb); #endif } #if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { Py_XINCREF(type); if (!value || value == Py_None) value = NULL; else Py_INCREF(value); if (!tb || tb == Py_None) tb = NULL; else { Py_INCREF(tb); if (!PyTraceBack_Check(tb)) { PyErr_SetString(PyExc_TypeError, "raise: arg 3 must be a traceback or None"); goto raise_error; } } #if PY_VERSION_HEX < 0x02050000 if (PyClass_Check(type)) { #else if (PyType_Check(type)) { #endif #if CYTHON_COMPILING_IN_PYPY if (!value) { Py_INCREF(Py_None); value = Py_None; } #endif PyErr_NormalizeException(&type, &value, &tb); } else { if (value) { PyErr_SetString(PyExc_TypeError, "instance exception may not have a separate value"); goto raise_error; } value = type; #if PY_VERSION_HEX < 0x02050000 if (PyInstance_Check(type)) { type = (PyObject*) ((PyInstanceObject*)type)->in_class; Py_INCREF(type); } else { type = 0; PyErr_SetString(PyExc_TypeError, "raise: exception must be an old-style class or instance"); goto raise_error; } #else type = (PyObject*) Py_TYPE(type); Py_INCREF(type); if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { PyErr_SetString(PyExc_TypeError, "raise: exception class must be a subclass of BaseException"); goto raise_error; } #endif } __Pyx_ErrRestore(type, value, tb); return; raise_error: Py_XDECREF(value); Py_XDECREF(type); Py_XDECREF(tb); return; } #else /* Python 3+ */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { PyObject* owned_instance = NULL; if (tb == Py_None) { tb = 0; } else if (tb && !PyTraceBack_Check(tb)) { PyErr_SetString(PyExc_TypeError, "raise: arg 3 must be a traceback or None"); goto bad; } if (value == Py_None) value = 0; if (PyExceptionInstance_Check(type)) { if (value) { PyErr_SetString(PyExc_TypeError, "instance exception may not have a separate value"); goto bad; } value = type; type = (PyObject*) Py_TYPE(value); } else if (PyExceptionClass_Check(type)) { PyObject *args; if (!value) args = PyTuple_New(0); else if (PyTuple_Check(value)) { Py_INCREF(value); args = value; } else args = PyTuple_Pack(1, value); if (!args) goto bad; owned_instance = PyEval_CallObject(type, args); Py_DECREF(args); if (!owned_instance) goto bad; value = owned_instance; if (!PyExceptionInstance_Check(value)) { PyErr_Format(PyExc_TypeError, "calling %R should have returned an instance of " "BaseException, not %R", type, Py_TYPE(value)); goto bad; } } else { PyErr_SetString(PyExc_TypeError, "raise: exception class must be a subclass of BaseException"); goto bad; } #if PY_VERSION_HEX >= 0x03030000 if (cause) { #else if (cause && cause != Py_None) { #endif PyObject *fixed_cause; if (cause == Py_None) { fixed_cause = NULL; } else if (PyExceptionClass_Check(cause)) { fixed_cause = PyObject_CallObject(cause, NULL); if (fixed_cause == NULL) goto bad; } else if (PyExceptionInstance_Check(cause)) { fixed_cause = cause; Py_INCREF(fixed_cause); } else { PyErr_SetString(PyExc_TypeError, "exception causes must derive from " "BaseException"); goto bad; } PyException_SetCause(value, fixed_cause); } PyErr_SetObject(type, value); if (tb) { PyThreadState *tstate = PyThreadState_GET(); PyObject* tmp_tb = tstate->curexc_traceback; if (tb != tmp_tb) { Py_INCREF(tb); tstate->curexc_traceback = tb; Py_XDECREF(tmp_tb); } } bad: Py_XDECREF(owned_instance); return; } #endif static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { PyObject *local_type, *local_value, *local_tb; #if CYTHON_COMPILING_IN_CPYTHON PyObject *tmp_type, *tmp_value, *tmp_tb; PyThreadState *tstate = PyThreadState_GET(); local_type = tstate->curexc_type; local_value = tstate->curexc_value; local_tb = tstate->curexc_traceback; tstate->curexc_type = 0; tstate->curexc_value = 0; tstate->curexc_traceback = 0; #else PyErr_Fetch(&local_type, &local_value, &local_tb); #endif PyErr_NormalizeException(&local_type, &local_value, &local_tb); #if CYTHON_COMPILING_IN_CPYTHON if (unlikely(tstate->curexc_type)) #else if (unlikely(PyErr_Occurred())) #endif goto bad; #if PY_MAJOR_VERSION >= 3 if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) goto bad; #endif Py_INCREF(local_type); Py_INCREF(local_value); Py_INCREF(local_tb); *type = local_type; *value = local_value; *tb = local_tb; #if CYTHON_COMPILING_IN_CPYTHON tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = local_type; tstate->exc_value = local_value; tstate->exc_traceback = local_tb; /* Make sure tstate is in a consistent state when we XDECREF these objects (DECREF may run arbitrary code). */ Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); #else PyErr_SetExcInfo(local_type, local_value, local_tb); #endif return 0; bad: *type = 0; *value = 0; *tb = 0; Py_XDECREF(local_type); Py_XDECREF(local_value); Py_XDECREF(local_tb); return -1; } static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { PyObject *r; if (!j) return NULL; r = PyObject_GetItem(o, j); Py_DECREF(j); return r; } static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, int wraparound, int boundscheck) { #if CYTHON_COMPILING_IN_CPYTHON if (wraparound & unlikely(i < 0)) i += PyList_GET_SIZE(o); if ((!boundscheck) || likely((0 <= i) & (i < PyList_GET_SIZE(o)))) { PyObject *r = PyList_GET_ITEM(o, i); Py_INCREF(r); return r; } return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); #else return PySequence_GetItem(o, i); #endif } static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, int wraparound, int boundscheck) { #if CYTHON_COMPILING_IN_CPYTHON if (wraparound & unlikely(i < 0)) i += PyTuple_GET_SIZE(o); if ((!boundscheck) || likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) { PyObject *r = PyTuple_GET_ITEM(o, i); Py_INCREF(r); return r; } return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); #else return PySequence_GetItem(o, i); #endif } static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck) { #if CYTHON_COMPILING_IN_CPYTHON if (is_list || PyList_CheckExact(o)) { Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); if ((!boundscheck) || (likely((n >= 0) & (n < PyList_GET_SIZE(o))))) { PyObject *r = PyList_GET_ITEM(o, n); Py_INCREF(r); return r; } } else if (PyTuple_CheckExact(o)) { Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); if ((!boundscheck) || likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) { PyObject *r = PyTuple_GET_ITEM(o, n); Py_INCREF(r); return r; } } else { PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; if (likely(m && m->sq_item)) { if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { Py_ssize_t l = m->sq_length(o); if (likely(l >= 0)) { i += l; } else { if (PyErr_ExceptionMatches(PyExc_OverflowError)) PyErr_Clear(); else return NULL; } } return m->sq_item(o, i); } } #else if (is_list || PySequence_Check(o)) { return PySequence_GetItem(o, i); } #endif return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); } static CYTHON_INLINE int __Pyx_CheckKeywordStrings( PyObject *kwdict, const char* function_name, int kw_allowed) { PyObject* key = 0; Py_ssize_t pos = 0; #if CPYTHON_COMPILING_IN_PYPY if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) goto invalid_keyword; return 1; #else while (PyDict_Next(kwdict, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) #endif if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; } if ((!kw_allowed) && unlikely(key)) goto invalid_keyword; return 1; invalid_keyword_type: PyErr_Format(PyExc_TypeError, "%s() keywords must be strings", function_name); return 0; #endif invalid_keyword: PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 "%s() got an unexpected keyword argument '%s'", function_name, PyString_AsString(key)); #else "%s() got an unexpected keyword argument '%U'", function_name, key); #endif return 0; } static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name) { PyObject *result; result = __Pyx_PyObject_GetAttrStr(nmspace, name); if (!result) result = __Pyx_GetModuleGlobalName(name); return result; } static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb) { #if CYTHON_COMPILING_IN_CPYTHON PyThreadState *tstate = PyThreadState_GET(); *type = tstate->exc_type; *value = tstate->exc_value; *tb = tstate->exc_traceback; Py_XINCREF(*type); Py_XINCREF(*value); Py_XINCREF(*tb); #else PyErr_GetExcInfo(type, value, tb); #endif } static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb) { #if CYTHON_COMPILING_IN_CPYTHON PyObject *tmp_type, *tmp_value, *tmp_tb; PyThreadState *tstate = PyThreadState_GET(); tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = type; tstate->exc_value = value; tstate->exc_traceback = tb; Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); #else PyErr_SetExcInfo(type, value, tb); #endif } static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; PyObject *module = 0; PyObject *global_dict = 0; PyObject *empty_dict = 0; PyObject *list; #if PY_VERSION_HEX < 0x03030000 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s____import__); if (!py_import) goto bad; #endif if (from_list) list = from_list; else { empty_list = PyList_New(0); if (!empty_list) goto bad; list = empty_list; } global_dict = PyModule_GetDict(__pyx_m); if (!global_dict) goto bad; empty_dict = PyDict_New(); if (!empty_dict) goto bad; #if PY_VERSION_HEX >= 0x02050000 { #if PY_MAJOR_VERSION >= 3 if (level == -1) { if (strchr(__Pyx_MODULE_NAME, '.')) { #if PY_VERSION_HEX < 0x03030000 PyObject *py_level = PyInt_FromLong(1); if (!py_level) goto bad; module = PyObject_CallFunctionObjArgs(py_import, name, global_dict, empty_dict, list, py_level, NULL); Py_DECREF(py_level); #else module = PyImport_ImportModuleLevelObject( name, global_dict, empty_dict, list, 1); #endif if (!module) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) goto bad; PyErr_Clear(); } } level = 0; /* try absolute import on failure */ } #endif if (!module) { #if PY_VERSION_HEX < 0x03030000 PyObject *py_level = PyInt_FromLong(level); if (!py_level) goto bad; module = PyObject_CallFunctionObjArgs(py_import, name, global_dict, empty_dict, list, py_level, NULL); Py_DECREF(py_level); #else module = PyImport_ImportModuleLevelObject( name, global_dict, empty_dict, list, level); #endif } } #else if (level>0) { PyErr_SetString(PyExc_RuntimeError, "Relative import is not supported for Python <=2.4."); goto bad; } module = PyObject_CallFunctionObjArgs(py_import, name, global_dict, empty_dict, list, NULL); #endif bad: #if PY_VERSION_HEX < 0x03030000 Py_XDECREF(py_import); #endif Py_XDECREF(empty_list); Py_XDECREF(empty_dict); return module; } static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { #if CYTHON_COMPILING_IN_PYPY if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) { /* cdef classes */ return PyClassMethod_New(method); } #else static PyTypeObject *methoddescr_type = NULL; if (methoddescr_type == NULL) { PyObject *meth = __Pyx_GetAttrString((PyObject*)&PyList_Type, "append"); if (!meth) return NULL; methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } if (PyObject_TypeCheck(method, methoddescr_type)) { /* cdef classes */ PyMethodDescrObject *descr = (PyMethodDescrObject *)method; #if PY_VERSION_HEX < 0x03020000 PyTypeObject *d_type = descr->d_type; #else PyTypeObject *d_type = descr->d_common.d_type; #endif return PyDescr_NewClassMethod(d_type, descr->d_method); } #endif else if (PyMethod_Check(method)) { /* python classes */ return PyClassMethod_New(PyMethod_GET_FUNCTION(method)); } else if (PyCFunction_Check(method)) { return PyClassMethod_New(method); } #ifdef __Pyx_CyFunction_USED else if (PyObject_TypeCheck(method, __pyx_CyFunctionType)) { return PyClassMethod_New(method); } #endif PyErr_Format(PyExc_TypeError, "Class-level classmethod() can only be called on " "a method_descriptor or instance method."); return NULL; } static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { const unsigned char neg_one = (unsigned char)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(unsigned char) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(unsigned char)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to unsigned char" : "value too large to convert to unsigned char"); } return (unsigned char)-1; } return (unsigned char)val; } return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); } static CYTHON_INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { const unsigned short neg_one = (unsigned short)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(unsigned short) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(unsigned short)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to unsigned short" : "value too large to convert to unsigned short"); } return (unsigned short)-1; } return (unsigned short)val; } return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); } static CYTHON_INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { const unsigned int neg_one = (unsigned int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(unsigned int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(unsigned int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to unsigned int" : "value too large to convert to unsigned int"); } return (unsigned int)-1; } return (unsigned int)val; } return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); } static CYTHON_INLINE char __Pyx_PyInt_AsChar(PyObject* x) { const char neg_one = (char)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(char) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(char)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to char" : "value too large to convert to char"); } return (char)-1; } return (char)val; } return (char)__Pyx_PyInt_AsLong(x); } static CYTHON_INLINE short __Pyx_PyInt_AsShort(PyObject* x) { const short neg_one = (short)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(short) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(short)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to short" : "value too large to convert to short"); } return (short)-1; } return (short)val; } return (short)__Pyx_PyInt_AsLong(x); } static CYTHON_INLINE int __Pyx_PyInt_AsInt(PyObject* x) { const int neg_one = (int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to int" : "value too large to convert to int"); } return (int)-1; } return (int)val; } return (int)__Pyx_PyInt_AsLong(x); } static CYTHON_INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { const signed char neg_one = (signed char)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(signed char) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(signed char)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to signed char" : "value too large to convert to signed char"); } return (signed char)-1; } return (signed char)val; } return (signed char)__Pyx_PyInt_AsSignedLong(x); } static CYTHON_INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { const signed short neg_one = (signed short)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(signed short) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(signed short)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to signed short" : "value too large to convert to signed short"); } return (signed short)-1; } return (signed short)val; } return (signed short)__Pyx_PyInt_AsSignedLong(x); } static CYTHON_INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { const signed int neg_one = (signed int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(signed int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(signed int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to signed int" : "value too large to convert to signed int"); } return (signed int)-1; } return (signed int)val; } return (signed int)__Pyx_PyInt_AsSignedLong(x); } static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject* x) { const int neg_one = (int)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; if (sizeof(int) < sizeof(long)) { long val = __Pyx_PyInt_AsLong(x); if (unlikely(val != (long)(int)val)) { if (!unlikely(val == -1 && PyErr_Occurred())) { PyErr_SetString(PyExc_OverflowError, (is_unsigned && unlikely(val < 0)) ? "can't convert negative value to int" : "value too large to convert to int"); } return (int)-1; } return (int)val; } return (int)__Pyx_PyInt_AsLong(x); } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { const unsigned long neg_one = (unsigned long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned long"); return (unsigned long)-1; } return (unsigned long)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (unsigned long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned long"); return (unsigned long)-1; } return (unsigned long)PyLong_AsUnsignedLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(unsigned long) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(unsigned long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (unsigned long)PyLong_AsLong(x); } } else { unsigned long val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (unsigned long)-1; val = __Pyx_PyInt_AsUnsignedLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { const unsigned PY_LONG_LONG neg_one = (unsigned PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned PY_LONG_LONG"); return (unsigned PY_LONG_LONG)-1; } return (unsigned PY_LONG_LONG)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned PY_LONG_LONG"); return (unsigned PY_LONG_LONG)-1; } return (unsigned PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(unsigned PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (unsigned PY_LONG_LONG)PyLong_AsLongLong(x); } } else { unsigned PY_LONG_LONG val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (unsigned PY_LONG_LONG)-1; val = __Pyx_PyInt_AsUnsignedLongLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { const long neg_one = (long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to long"); return (long)-1; } return (long)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to long"); return (long)-1; } return (long)PyLong_AsUnsignedLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(long) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (long)PyLong_AsLong(x); } } else { long val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (long)-1; val = __Pyx_PyInt_AsLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { const PY_LONG_LONG neg_one = (PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to PY_LONG_LONG"); return (PY_LONG_LONG)-1; } return (PY_LONG_LONG)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to PY_LONG_LONG"); return (PY_LONG_LONG)-1; } return (PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (PY_LONG_LONG)PyLong_AsLongLong(x); } } else { PY_LONG_LONG val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (PY_LONG_LONG)-1; val = __Pyx_PyInt_AsLongLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { const signed long neg_one = (signed long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed long"); return (signed long)-1; } return (signed long)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (signed long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed long"); return (signed long)-1; } return (signed long)PyLong_AsUnsignedLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed long)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(signed long) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(signed long) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (signed long)PyLong_AsLong(x); } } else { signed long val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (signed long)-1; val = __Pyx_PyInt_AsSignedLong(tmp); Py_DECREF(tmp); return val; } } #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #endif #endif static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { const signed PY_LONG_LONG neg_one = (signed PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed PY_LONG_LONG"); return (signed PY_LONG_LONG)-1; } return (signed PY_LONG_LONG)val; } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return (signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed PY_LONG_LONG"); return (signed PY_LONG_LONG)-1; } return (signed PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 #if CYTHON_USE_PYLONG_INTERNALS if (sizeof(digit) <= sizeof(signed PY_LONG_LONG)) { switch (Py_SIZE(x)) { case 0: return 0; case 1: return +(signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; case -1: return -(signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; } } #endif #endif return (signed PY_LONG_LONG)PyLong_AsLongLong(x); } } else { signed PY_LONG_LONG val; PyObject *tmp = __Pyx_PyNumber_Int(x); if (!tmp) return (signed PY_LONG_LONG)-1; val = __Pyx_PyInt_AsSignedLongLong(tmp); Py_DECREF(tmp); return val; } } static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { char message[200]; PyOS_snprintf(message, sizeof(message), "compiletime version %s of module '%.100s' " "does not match runtime version %s", ctversion, __Pyx_MODULE_NAME, rtversion); #if PY_VERSION_HEX < 0x02050000 return PyErr_Warn(NULL, message); #else return PyErr_WarnEx(NULL, message, 1); #endif } return 0; } static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { return count; } while (start < end) { mid = (start + end) / 2; if (code_line < entries[mid].code_line) { end = mid; } else if (code_line > entries[mid].code_line) { start = mid + 1; } else { return mid; } } if (code_line <= entries[mid].code_line) { return mid; } else { return mid + 1; } } static PyCodeObject *__pyx_find_code_object(int code_line) { PyCodeObject* code_object; int pos; if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { return NULL; } pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { return NULL; } code_object = __pyx_code_cache.entries[pos].code_object; Py_INCREF(code_object); return code_object; } static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { int pos, i; __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; if (unlikely(!code_line)) { return; } if (unlikely(!entries)) { entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); if (likely(entries)) { __pyx_code_cache.entries = entries; __pyx_code_cache.max_count = 64; __pyx_code_cache.count = 1; entries[0].code_line = code_line; entries[0].code_object = code_object; Py_INCREF(code_object); } return; } pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { PyCodeObject* tmp = entries[pos].code_object; entries[pos].code_object = code_object; Py_DECREF(tmp); return; } if (__pyx_code_cache.count == __pyx_code_cache.max_count) { int new_max = __pyx_code_cache.max_count + 64; entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( __pyx_code_cache.entries, new_max*sizeof(__Pyx_CodeObjectCacheEntry)); if (unlikely(!entries)) { return; } __pyx_code_cache.entries = entries; __pyx_code_cache.max_count = new_max; } for (i=__pyx_code_cache.count; i>pos; i--) { entries[i] = entries[i-1]; } entries[pos].code_line = code_line; entries[pos].code_object = code_object; __pyx_code_cache.count++; Py_INCREF(code_object); } #include "compile.h" #include "frameobject.h" #include "traceback.h" static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyObject *py_srcfile = 0; PyObject *py_funcname = 0; #if PY_MAJOR_VERSION < 3 py_srcfile = PyString_FromString(filename); #else py_srcfile = PyUnicode_FromString(filename); #endif if (!py_srcfile) goto bad; if (c_line) { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); #else py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); #endif } else { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromString(funcname); #else py_funcname = PyUnicode_FromString(funcname); #endif } if (!py_funcname) goto bad; py_code = __Pyx_PyCode_New( 0, /*int argcount,*/ 0, /*int kwonlyargcount,*/ 0, /*int nlocals,*/ 0, /*int stacksize,*/ 0, /*int flags,*/ __pyx_empty_bytes, /*PyObject *code,*/ __pyx_empty_tuple, /*PyObject *consts,*/ __pyx_empty_tuple, /*PyObject *names,*/ __pyx_empty_tuple, /*PyObject *varnames,*/ __pyx_empty_tuple, /*PyObject *freevars,*/ __pyx_empty_tuple, /*PyObject *cellvars,*/ py_srcfile, /*PyObject *filename,*/ py_funcname, /*PyObject *name,*/ py_line, /*int firstlineno,*/ __pyx_empty_bytes /*PyObject *lnotab*/ ); Py_DECREF(py_srcfile); Py_DECREF(py_funcname); return py_code; bad: Py_XDECREF(py_srcfile); Py_XDECREF(py_funcname); return NULL; } static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyObject *py_globals = 0; PyFrameObject *py_frame = 0; py_code = __pyx_find_code_object(c_line ? c_line : py_line); if (!py_code) { py_code = __Pyx_CreateCodeObjectForTraceback( funcname, c_line, py_line, filename); if (!py_code) goto bad; __pyx_insert_code_object(c_line ? c_line : py_line, py_code); } py_globals = PyModule_GetDict(__pyx_m); if (!py_globals) goto bad; py_frame = PyFrame_New( PyThreadState_GET(), /*PyThreadState *tstate,*/ py_code, /*PyCodeObject *code,*/ py_globals, /*PyObject *globals,*/ 0 /*PyObject *locals*/ ); if (!py_frame) goto bad; py_frame->f_lineno = py_line; PyTraceBack_Here(py_frame); bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { *t->p = PyString_InternFromString(t->s); } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } #else /* Python 3+ has unicode identifiers */ if (t->is_unicode | t->is_str) { if (t->intern) { *t->p = PyUnicode_InternFromString(t->s); } else if (t->encoding) { *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); } else { *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); } } else { *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); } #endif if (!*t->p) return -1; ++t; } return 0; } static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char* c_str) { return __Pyx_PyUnicode_FromStringAndSize(c_str, strlen(c_str)); } static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { Py_ssize_t ignore; return __Pyx_PyObject_AsStringAndSize(o, &ignore); } static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT if ( #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII __Pyx_sys_getdefaultencoding_not_ascii && #endif PyUnicode_Check(o)) { #if PY_VERSION_HEX < 0x03030000 char* defenc_c; PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); if (!defenc) return NULL; defenc_c = PyBytes_AS_STRING(defenc); #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII { char* end = defenc_c + PyBytes_GET_SIZE(defenc); char* c; for (c = defenc_c; c < end; c++) { if ((unsigned char) (*c) >= 128) { PyUnicode_AsASCIIString(o); return NULL; } } } #endif /*__PYX_DEFAULT_STRING_ENCODING_IS_ASCII*/ *length = PyBytes_GET_SIZE(defenc); return defenc_c; #else /* PY_VERSION_HEX < 0x03030000 */ if (PyUnicode_READY(o) == -1) return NULL; #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII if (PyUnicode_IS_ASCII(o)) { *length = PyUnicode_GET_DATA_SIZE(o); return PyUnicode_AsUTF8(o); } else { PyUnicode_AsASCIIString(o); return NULL; } #else /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ return PyUnicode_AsUTF8AndSize(o, length); #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ #endif /* PY_VERSION_HEX < 0x03030000 */ } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ { char* result; int r = PyBytes_AsStringAndSize(o, &result, length); if (r < 0) { return NULL; } else { return result; } } } static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { int is_true = x == Py_True; if (is_true | (x == Py_False) | (x == Py_None)) return is_true; else return PyObject_IsTrue(x); } static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { PyNumberMethods *m; const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 if (PyInt_Check(x) || PyLong_Check(x)) #else if (PyLong_Check(x)) #endif return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } else if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } #else if (m && m->nb_int) { name = "int"; res = PyNumber_Long(x); } #endif if (res) { #if PY_MAJOR_VERSION < 3 if (!PyInt_Check(res) && !PyLong_Check(res)) { #else if (!PyLong_Check(res)) { #endif PyErr_Format(PyExc_TypeError, "__%s__ returned non-%s (type %.200s)", name, name, Py_TYPE(res)->tp_name); Py_DECREF(res); return NULL; } } else if (!PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "an integer is required"); } return res; } static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { Py_ssize_t ival; PyObject* x = PyNumber_Index(b); if (!x) return -1; ival = PyInt_AsSsize_t(x); Py_DECREF(x); return ival; } static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { #if PY_VERSION_HEX < 0x02050000 if (ival <= LONG_MAX) return PyInt_FromLong((long)ival); else { unsigned char *bytes = (unsigned char *) &ival; int one = 1; int little = (int)*(unsigned char*)&one; return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); } #else return PyInt_FromSize_t(ival); #endif } static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { if ((val != (unsigned PY_LONG_LONG)-1) || !PyErr_Occurred()) PyErr_SetString(PyExc_OverflowError, "value too large to convert to size_t"); return (size_t)-1; } return (size_t)val; } #endif /* Py_PYTHON_H */ Attic-0.10/attic/_hashindex.c0000644000175000017500000002506112272262001016324 0ustar jonasjonas00000000000000#include #include #include #include #include #include #include #include #include #include #if defined(BYTE_ORDER)&&(BYTE_ORDER == BIG_ENDIAN) #define _le32toh(x) __builtin_bswap32(x) #define _htole32(x) __builtin_bswap32(x) #elif defined(BYTE_ORDER)&&(BYTE_ORDER == LITTLE_ENDIAN) #define _le32toh(x) (x) #define _htole32(x) (x) #else #error Unknown byte order #endif typedef struct { char magic[8]; int32_t num_entries; int32_t num_buckets; int8_t key_size; int8_t value_size; } __attribute__((__packed__)) HashHeader; typedef struct { char *path; void *map_addr; off_t map_length; void *buckets; int num_entries; int num_buckets; int key_size; int value_size; int bucket_size; int lower_limit; int upper_limit; int readonly; } HashIndex; #define MAGIC "ATTICIDX" #define EMPTY _htole32(0xffffffff) #define DELETED _htole32(0xfffffffe) #define MAX_BUCKET_SIZE 512 #define BUCKET_LOWER_LIMIT .25 #define BUCKET_UPPER_LIMIT .90 #define MIN_BUCKETS 1024 #define MAX(x, y) ((x) > (y) ? (x): (y)) #define BUCKET_ADDR(index, idx) (index->buckets + (idx * index->bucket_size)) #define BUCKET_IS_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) == DELETED) #define BUCKET_IS_EMPTY(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) == EMPTY) #define BUCKET_MATCHES_KEY(index, idx, key) (memcmp(key, BUCKET_ADDR(index, idx), index->key_size) == 0) #define BUCKET_MARK_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = DELETED) #define EPRINTF(msg, ...) EPRINTF_PATH(index->path, msg, ##__VA_ARGS__) #define EPRINTF_PATH(path, msg, ...) fprintf(stderr, "hashindex: %s: " msg "\n", path, ##__VA_ARGS__) static HashIndex *hashindex_open(const char *path, int readonly); static int hashindex_close(HashIndex *index); static int hashindex_clear(HashIndex *index); static int hashindex_flush(HashIndex *index); static HashIndex *hashindex_create(const char *path, int capacity, int key_size, int value_size); static const void *hashindex_get(HashIndex *index, const void *key); static int hashindex_set(HashIndex *index, const void *key, const void *value); static int hashindex_delete(HashIndex *index, const void *key); static void *hashindex_next_key(HashIndex *index, const void *key); /* Private API */ static int hashindex_index(HashIndex *index, const void *key) { return _le32toh(*((uint32_t *)key)) % index->num_buckets; } static int hashindex_lookup(HashIndex *index, const void *key) { int didx = -1; int start = hashindex_index(index, key); int idx = start; for(;;) { if(BUCKET_IS_EMPTY(index, idx)) { return -1; } if(BUCKET_IS_DELETED(index, idx)) { if(didx == -1) { didx = idx; } } else if(BUCKET_MATCHES_KEY(index, idx, key)) { if (didx != -1 && !index->readonly) { memcpy(BUCKET_ADDR(index, didx), BUCKET_ADDR(index, idx), index->bucket_size); BUCKET_MARK_DELETED(index, idx); idx = didx; } return idx; } idx = (idx + 1) % index->num_buckets; if(idx == start) { return -1; } } } static int hashindex_resize(HashIndex *index, int capacity) { char *new_path = malloc(strlen(index->path) + 5); int ret = 0; strcpy(new_path, index->path); strcat(new_path, ".tmp"); HashIndex *new; if(!(new = hashindex_create(new_path, capacity, index->key_size, index->value_size))) { free(new_path); return 0; } void *key = NULL; while((key = hashindex_next_key(index, key))) { hashindex_set(new, key, hashindex_get(index, key)); } munmap(index->map_addr, index->map_length); index->map_addr = new->map_addr; index->map_length = new->map_length; index->num_buckets = new->num_buckets; index->lower_limit = new->lower_limit; index->upper_limit = new->upper_limit; index->buckets = new->buckets; if(unlink(index->path) < 0) { EPRINTF("unlink failed"); goto out; } if(rename(new_path, index->path) < 0) { EPRINTF_PATH(new_path, "rename failed"); goto out; } ret = 1; out: free(new_path); free(new->path); free(new); return ret; } /* Public API */ static HashIndex * hashindex_open(const char *path, int readonly) { void *addr; int fd, oflags, prot; off_t length; HashHeader *header; HashIndex *index; if(readonly) { oflags = O_RDONLY; prot = PROT_READ; } else { oflags = O_RDWR; prot = PROT_READ | PROT_WRITE; } if((fd = open(path, oflags)) < 0) { EPRINTF_PATH(path, "open failed"); fprintf(stderr, "Failed to open %s\n", path); return NULL; } if((length = lseek(fd, 0, SEEK_END)) < 0) { EPRINTF_PATH(path, "lseek failed"); if(close(fd) < 0) { EPRINTF_PATH(path, "close failed"); } return NULL; } addr = mmap(0, length, prot, MAP_SHARED, fd, 0); if(close(fd) < 0) { EPRINTF_PATH(path, "close failed"); return NULL; } if(addr == MAP_FAILED) { EPRINTF_PATH(path, "mmap failed"); return NULL; } header = (HashHeader *)addr; if(memcmp(header->magic, MAGIC, 8)) { EPRINTF_PATH(path, "Unknown file header"); return NULL; } if(length != sizeof(HashHeader) + _le32toh(header->num_buckets) * (header->key_size + header->value_size)) { EPRINTF_PATH(path, "Incorrect file length"); return NULL; } if(!(index = malloc(sizeof(HashIndex)))) { EPRINTF_PATH(path, "malloc failed"); return NULL; } index->readonly = readonly; index->map_addr = addr; index->map_length = length; index->num_entries = _le32toh(header->num_entries); index->num_buckets = _le32toh(header->num_buckets); index->key_size = header->key_size; index->value_size = header->value_size; index->bucket_size = index->key_size + index->value_size; index->buckets = (addr + sizeof(HashHeader)); index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0; index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT); if(!(index->path = strdup(path))) { EPRINTF_PATH(path, "strdup failed"); free(index); return NULL; } return index; } static HashIndex * hashindex_create(const char *path, int capacity, int key_size, int value_size) { FILE *fd; char bucket[MAX_BUCKET_SIZE] = {}; int i, bucket_size; HashHeader header = { .magic = MAGIC, .num_entries = 0, .key_size = key_size, .value_size = value_size }; capacity = MAX(MIN_BUCKETS, capacity); header.num_buckets = _htole32(capacity); if(!(fd = fopen(path, "w"))) { EPRINTF_PATH(path, "fopen failed"); return NULL; } bucket_size = key_size + value_size; if(fwrite(&header, 1, sizeof(header), fd) != sizeof(header)) { goto error; } *((uint32_t *)(bucket + key_size)) = EMPTY; for(i = 0; i < capacity; i++) { if(fwrite(bucket, 1, bucket_size, fd) != bucket_size) { goto error; } } if(fclose(fd) < 0) { EPRINTF_PATH(path, "fclose failed"); if(unlink(path) < 0) { EPRINTF_PATH(path, "unlink failed"); } return NULL; } return hashindex_open(path, 0); error: if(unlink(path) < 0) { EPRINTF_PATH(path, "unlink failed"); } EPRINTF_PATH(path, "fwrite failed"); if(fclose(fd) < 0) { EPRINTF_PATH(path, "fclose failed"); } return NULL; } static int hashindex_clear(HashIndex *index) { int i; for(i = 0; i < index->num_buckets; i++) { BUCKET_MARK_DELETED(index, i); } index->num_entries = 0; return hashindex_resize(index, MIN_BUCKETS); } static int hashindex_flush(HashIndex *index) { if(index->readonly) { return 1; } *((uint32_t *)(index->map_addr + 8)) = _htole32(index->num_entries); *((uint32_t *)(index->map_addr + 12)) = _htole32(index->num_buckets); if(msync(index->map_addr, index->map_length, MS_SYNC) < 0) { EPRINTF("msync failed"); return 0; } return 1; } static int hashindex_close(HashIndex *index) { int rv = 1; if(hashindex_flush(index) < 0) { rv = 0; } if(munmap(index->map_addr, index->map_length) < 0) { EPRINTF("munmap failed"); rv = 0; } free(index->path); free(index); return rv; } static const void * hashindex_get(HashIndex *index, const void *key) { int idx = hashindex_lookup(index, key); if(idx < 0) { return NULL; } return BUCKET_ADDR(index, idx) + index->key_size; } static int hashindex_set(HashIndex *index, const void *key, const void *value) { int idx = hashindex_lookup(index, key); uint8_t *ptr; if(idx < 0) { if(index->num_entries > index->upper_limit) { if(!hashindex_resize(index, index->num_buckets * 2)) { return 0; } } idx = hashindex_index(index, key); while(!BUCKET_IS_EMPTY(index, idx) && !BUCKET_IS_DELETED(index, idx)) { idx = (idx + 1) % index->num_buckets; } ptr = BUCKET_ADDR(index, idx); memcpy(ptr, key, index->key_size); memcpy(ptr + index->key_size, value, index->value_size); index->num_entries += 1; } else { memcpy(BUCKET_ADDR(index, idx) + index->key_size, value, index->value_size); } return 1; } static int hashindex_delete(HashIndex *index, const void *key) { int idx = hashindex_lookup(index, key); if (idx < 0) { return 1; } BUCKET_MARK_DELETED(index, idx); index->num_entries -= 1; if(index->num_entries < index->lower_limit) { if(!hashindex_resize(index, index->num_buckets / 2)) { return 0; } } return 1; } static void * hashindex_next_key(HashIndex *index, const void *key) { int idx = 0; if(key) { idx = 1 + (key - index->buckets) / index->bucket_size; } if (idx == index->num_buckets) { return NULL; } while(BUCKET_IS_EMPTY(index, idx) || BUCKET_IS_DELETED(index, idx)) { idx ++; if (idx == index->num_buckets) { return NULL; } } return BUCKET_ADDR(index, idx); } static int hashindex_get_size(HashIndex *index) { return index->num_entries; } Attic-0.10/attic/_version.py0000644000175000017500000000064012272541113016244 0ustar jonasjonas00000000000000 # This file was generated by 'versioneer.py' (0.7+) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. version_version = '0.10' version_full = '50cabd53b1c66f4c24c4cb92d5d00d6a4dd355c5' def get_versions(default={}, verbose=False): return {'version': version_version, 'full': version_full} Attic-0.10/docs/0000755000175000017500000000000012272541113013672 5ustar jonasjonas00000000000000Attic-0.10/docs/_themes/0000755000175000017500000000000012272541113015316 5ustar jonasjonas00000000000000Attic-0.10/docs/_themes/attic/0000755000175000017500000000000012272541113016422 5ustar jonasjonas00000000000000Attic-0.10/docs/_themes/attic/static/0000755000175000017500000000000012272541113017711 5ustar jonasjonas00000000000000Attic-0.10/docs/_themes/attic/static/attic.css_t0000644000175000017500000000403212177534770022070 0ustar jonasjonas00000000000000@import url("basic.css"); body { font-family: Arial, Sans-Serif; background-color: white; margin: 0 0 4em; padding: 0; } div.related { display: none; background-color: black; padding: .4em; width: 800px; margin: 0 auto; } div.related a { color: white; text-decoration: none; } div.document { background-color: white; width: 850px; margin: 0 auto; padding-top: 1em; } div.documentwrapper { float: right; width: 600px; } div.sphinxsidebar { margin-left: 0; parring-right: 20px; width: 230px; } h1, h2, h3 { font-family: "DejaVu Serif"; font-weight: normal; font-style: italic; color: #333; } h1 { margin: .8em 0 .5em; } h2, h3 { margin: 1.2em 0 .6em; } h1 { font-size: 260%;} h2 { font-size: 180%;} h3 { font-size: 130%;} ul { padding-left: 1.2em; margin-bottom: .3em; } ul ul { font-size: 95%; } li { margin: .1em 0; } a:link, a:visited { color: #00608f; text-decoration: none; } a:hover { color: #00B0E4; border-bottom: 1px dotted #00B0E4; } div.sphinxsidebar a:link, div.sphinxsidebar a:visited { color: #555; border-bottom: 1px dotted #555; } div.sphinxsidebar input { border: 1px solid #ccc; } pre { border: 1px solid #ccc; padding: .6em; background: black; color: #ddd; border-radius: .4em; box-shadow: 2px 2px #ddd; line-height: 1.2em; } pre a:link, pre a:visited { color: #00B0E4; } div.sidebarlogo .title { font-family: "DejaVu Serif"; font-style: italic; font-size: 500%; } div.sidebarlogo .subtitle { font-style: italic; color: #777; } tt span.pre { font-size: 110%; } dt { font-style: italic; font-size: 95%; } div.admonition p.admonition-title + p { display: inline; } div.admonition p { margin-bottom: 5px; } p.admonition-title { display: inline; } p.admonition-title:after { content: ":"; } div.note { background-color: #eee; border: 1px solid #ccc; border-radius: .4em; box-shadow: 2px 2px #ddd; } div.seealso { background-color: #ffe; border: 1px solid #ff6; border-radius: .4em; box-shadow: 2px 2px #dd6; } Attic-0.10/docs/_themes/attic/sidebarlogo.html0000644000175000017500000000015112177161742021611 0ustar jonasjonas00000000000000Attic-0.10/docs/_themes/attic/sidebarusefullinks.html0000644000175000017500000000103012272524056023207 0ustar jonasjonas00000000000000Fork me on GitHub

Useful Links

Attic-0.10/docs/_themes/attic/theme.conf0000644000175000017500000000012112177161731020375 0ustar jonasjonas00000000000000[theme] inherit = basic stylesheet = attic.css pygments_style = tango [options] Attic-0.10/docs/usage/0000755000175000017500000000000012272541113014776 5ustar jonasjonas00000000000000Attic-0.10/docs/usage/change-passphrase.rst.inc0000644000175000017500000000057312272540714021707 0ustar jonasjonas00000000000000.. _attic_change-passphrase: attic change-passphrase ----------------------- :: usage: attic change-passphrase [-h] [-v] REPOSITORY Change repository key file passphrase positional arguments: REPOSITORY optional arguments: -h, --help show this help message and exit -v, --verbose verbose output Description ~~~~~~~~~~~ Attic-0.10/docs/usage/create.rst.inc0000644000175000017500000000174612272540715017562 0ustar jonasjonas00000000000000.. _attic_create: attic create ------------ :: usage: attic create [-h] [-v] [-s] [-e PATTERN] [-c SECONDS] [--do-not-cross-mountpoints] [--numeric-owner] ARCHIVE PATH [PATH ...] Create new archive positional arguments: ARCHIVE archive to create PATH paths to archive optional arguments: -h, --help show this help message and exit -v, --verbose verbose output -s, --stats print statistics for the created archive -e PATTERN, --exclude PATTERN exclude paths matching PATTERN -c SECONDS, --checkpoint-interval SECONDS write checkpoint every SECONDS seconds (Default: 300) --do-not-cross-mountpoints do not cross mount points --numeric-owner only store numeric user and group identifiers Description ~~~~~~~~~~~ Attic-0.10/docs/usage/delete.rst.inc0000644000175000017500000000051312272540715017550 0ustar jonasjonas00000000000000.. _attic_delete: attic delete ------------ :: usage: attic delete [-h] [-v] ARCHIVE Delete archive positional arguments: ARCHIVE archive to delete optional arguments: -h, --help show this help message and exit -v, --verbose verbose output Description ~~~~~~~~~~~ Attic-0.10/docs/usage/extract.rst.inc0000644000175000017500000000121612272540715017761 0ustar jonasjonas00000000000000.. _attic_extract: attic extract ------------- :: usage: attic extract [-h] [-v] [-e PATTERN] [--numeric-owner] ARCHIVE [PATH [PATH ...]] Extract archive contents positional arguments: ARCHIVE archive to extract PATH paths to extract optional arguments: -h, --help show this help message and exit -v, --verbose verbose output -e PATTERN, --exclude PATTERN exclude paths matching PATTERN --numeric-owner only obey numeric user and group identifiers Description ~~~~~~~~~~~ Attic-0.10/docs/usage/info.rst.inc0000644000175000017500000000056412272540715017247 0ustar jonasjonas00000000000000.. _attic_info: attic info ---------- :: usage: attic info [-h] [-v] ARCHIVE Show archive details such as disk space used positional arguments: ARCHIVE archive to display information about optional arguments: -h, --help show this help message and exit -v, --verbose verbose output Description ~~~~~~~~~~~ Attic-0.10/docs/usage/init.rst.inc0000644000175000017500000000101512272540715017247 0ustar jonasjonas00000000000000.. _attic_init: attic init ---------- :: usage: attic init [-h] [-v] [-e {none,passphrase,keyfile}] REPOSITORY Initialize an empty repository positional arguments: REPOSITORY repository to create optional arguments: -h, --help show this help message and exit -v, --verbose verbose output -e {none,passphrase,keyfile}, --encryption {none,passphrase,keyfile} select encryption method Description ~~~~~~~~~~~ Attic-0.10/docs/usage/list.rst.inc0000644000175000017500000000065412272540715017267 0ustar jonasjonas00000000000000.. _attic_list: attic list ---------- :: usage: attic list [-h] [-v] REPOSITORY_OR_ARCHIVE List archive or repository contents positional arguments: REPOSITORY_OR_ARCHIVE repository/archive to list contents of optional arguments: -h, --help show this help message and exit -v, --verbose verbose output Description ~~~~~~~~~~~ Attic-0.10/docs/usage/mount.rst.inc0000644000175000017500000000103212272540715017445 0ustar jonasjonas00000000000000.. _attic_mount: attic mount ----------- :: usage: attic mount [-h] [-v] [-f] [-o OPTIONS] ARCHIVE MOUNTPOINT Mount archive as a FUSE fileystem positional arguments: ARCHIVE archive to mount MOUNTPOINT where to mount filesystem optional arguments: -h, --help show this help message and exit -v, --verbose verbose output -f, --foreground stay in foreground, do not daemonize -o OPTIONS Extra mount options Description ~~~~~~~~~~~ Attic-0.10/docs/usage/prune.rst.inc0000644000175000017500000000211412272540716017437 0ustar jonasjonas00000000000000.. _attic_prune: attic prune ----------- :: usage: attic prune [-h] [-v] [-H HOURLY] [-d DAILY] [-w WEEKLY] [-m MONTHLY] [-y YEARLY] [-p PREFIX] REPOSITORY Prune repository archives according to specified rules positional arguments: REPOSITORY repository to prune optional arguments: -h, --help show this help message and exit -v, --verbose verbose output -H HOURLY, --hourly HOURLY number of hourly archives to keep -d DAILY, --daily DAILY number of daily archives to keep -w WEEKLY, --weekly WEEKLY number of daily archives to keep -m MONTHLY, --monthly MONTHLY number of monthly archives to keep -y YEARLY, --yearly YEARLY number of yearly archives to keep -p PREFIX, --prefix PREFIX only consider archive names starting with this prefix Description ~~~~~~~~~~~ Attic-0.10/docs/usage/verify.rst.inc0000644000175000017500000000104312272540716017612 0ustar jonasjonas00000000000000.. _attic_verify: attic verify ------------ :: usage: attic verify [-h] [-v] [-e PATTERN] ARCHIVE [PATH [PATH ...]] Verify archive consistency positional arguments: ARCHIVE archive to verity integrity of PATH paths to verify optional arguments: -h, --help show this help message and exit -v, --verbose verbose output -e PATTERN, --exclude PATTERN exclude paths matching PATTERN Description ~~~~~~~~~~~ Attic-0.10/docs/Makefile0000644000175000017500000001166212177161731015350 0ustar jonasjonas00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: ./update_usage.sh $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/attic.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/attic.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/attic" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/attic" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." make -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." gh-pages: html GH_PAGES_CLONE="`mktemp -d`" && \ git clone --branch gh-pages `git rev-parse --show-toplevel` $$GH_PAGES_CLONE && \ (cd $$GH_PAGES_CLONE && git rm -r *) && \ cp -r _build/html/* $$GH_PAGES_CLONE && \ (cd $$GH_PAGES_CLONE && git add -A && git commit -m 'Updated gh-pages' && git push) && \ rm -rf $$GH_PAGES_CLONE inotify: html while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done Attic-0.10/docs/conf.py0000644000175000017500000001605612261537600015205 0ustar jonasjonas00000000000000# -*- coding: utf-8 -*- # # Attic documentation build configuration file, created by # sphinx-quickstart on Sat Sep 10 18:18:25 2011. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os, attic # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = 'Attic - Deduplicating Archiver' copyright = '2010-2014, Jonas Borgström' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = attic.__version__.split('-')[0] # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'attic' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. html_theme_path = ['_themes'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = { 'index': ['sidebarlogo.html', 'sidebarusefullinks.html', 'searchbox.html'], '**': ['sidebarlogo.html', 'localtoc.html', 'relations.html', 'sidebarusefullinks.html', 'searchbox.html'] } # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = False # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. html_show_copyright = False # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'atticdoc' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'Attic.tex', 'Attic Documentation', 'Jonas Borgström', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). #man_pages = [ # ('man', 'attic', 'Attic', # ['Jonas Borgström'], 1) #] Attic-0.10/docs/faq.rst0000644000175000017500000000212312177161742015203 0ustar jonasjonas00000000000000.. _faq: .. include:: global.rst.inc Frequently asked questions ========================== Which platforms are supported? Currently Linux, FreeBSD and MacOS X are supported. Can I backup VM disk images? Yes, the :ref:`deduplication ` technique used by |project_name| makes sure only the modified parts of the file are stored. Which file attributes are preserved? The following attributes are preserved: * Name * Contents * Time of last modification (nanosecond precision with Python >= 3.3) * User ID of owner * Group ID of owner * Unix Permission * Extended attributes (xattrs) .. Note:: POSIX Access Control Lists (ACL_) are not yet preserved. How can I specify the encryption passphrase programmatically? The encryption passphrase can be specified programmatically using the `ATTIC_PASSPHRASE` environment variable. This is convenient when setting up automated encrypted backups. Another option is to use key file based encryption with a blank passphrase. See :ref:`encrypted_repos` for more details. Attic-0.10/docs/foreword.rst0000644000175000017500000000376612177542626016305 0ustar jonasjonas00000000000000.. include:: global.rst.inc .. _foreword: Foreword ======== |project_name| is a secure backup program for Linux, FreeBSD and Mac OS X. |project_name| is designed for efficient data storage where only new or modified data is stored. Features -------- Space efficient storage Variable block size `deduplication`_ is used to reduce the number of bytes stored by detecting redundant data. Each file is split into a number of variable length chunks and only chunks that have never been seen before are compressed and added to the repository. Optional data encryption All data can be protected using 256-bit AES_ encryption and data integrity and authenticity is verified using `HMAC-SHA256`_. Off-site backups |project_name| can store data on any remote host accessible over SSH as long as |project_name| is installed. Backups mountable as filesystems Backup archives are :ref:`mountable ` as `userspace filesystems`_ for easy backup verification and restores. Glossary -------- .. _deduplication_def: Deduplication Deduplication is a technique for improving storage utilization by eliminating redundant data. .. _archive_def: Archive An archive is a collection of files along with metadata that include file permissions, directory structure and various file attributes. Since each archive in a repository must have a unique name a good naming convention is ``hostname-YYYY-MM-DD``. .. _repository_def: Repository A repository is a filesystem directory storing data from zero or more archives. The data in a repository is both deduplicated and optionally encrypted making it both efficient and safe. Repositories are created using :ref:`attic_init` and the contents can be listed using :ref:`attic_list`. Key file When a repository is initialized a key file containing a password protected encryption key is created. It is vital to keep this file safe since the repository data is totally inaccessible without it. Attic-0.10/docs/global.rst.inc0000644000175000017500000000225012271172436016442 0ustar jonasjonas00000000000000.. highlight:: bash .. |project_name| replace:: ``Attic`` .. |package_dirname| replace:: Attic-|version| .. |package_filename| replace:: |package_dirname|.tar.gz .. |package_url| replace:: https://pypi.python.org/packages/source/A/Attic/|package_filename| .. |git_url| replace:: https://github.com/jborg/attic.git .. _deduplication: https://en.wikipedia.org/wiki/Data_deduplication .. _AES: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard .. _HMAC-SHA256: http://en.wikipedia.org/wiki/HMAC .. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2 .. _ACL: https://en.wikipedia.org/wiki/Access_control_list .. _github: https://github.com/jborg/attic .. _OpenSSL: https://www.openssl.org/ .. _Python: http://www.python.org/ .. _`msgpack-python`: https://pypi.python.org/pypi/msgpack-python/ .. _homebrew: http://mxcl.github.io/homebrew/ .. _issue tracker: https://github.com/jborg/attic/issues .. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace .. _librelist: http://librelist.com/ .. _Debian: http://packages.debian.org/attic .. _Ubuntu: http://packages.ubuntu.com/attic .. _Arch Linux: https://aur.archlinux.org/packages/attic/ .. _Cython: http://cython.org/ Attic-0.10/docs/index.rst0000644000175000017500000000335712271171027015545 0ustar jonasjonas00000000000000.. include:: global.rst.inc Welcome to Attic ================ |project_name| is a deduplicating backup program written in Python. The main goal of |project_name| is to provide an efficient and secure way to backup data. The data deduplication technique used makes |project_name| suitable for daily backups since only actual changes are stored. Easy to use ----------- Initialize a new backup :ref:`repository ` and create your first backup :ref:`archive ` in two lines:: $ attic init /usbdrive/my-backup.attic $ attic create -v /usbdrive/my-backup.attic::documents ~/Documents See the :ref:`quickstart` chapter for a more detailed example. Easy installation ----------------- You can use pip to install |project_name| quickly and easily:: $ pip install attic |project_name| is also part of the Debian_, Ubuntu_ and `Arch Linux`_ distributions of GNU/Linux. Need more help with installing? See :ref:`installation`. User's Guide ============ .. toctree:: :maxdepth: 2 foreword installation quickstart usage faq Getting help ============ If you've found a bug or have a concrete feature request you can add your bug report or feature request directly to the project `issue tracker`_. For more general questions or discussions a post to the mailing list is preferred. Mailing list ------------ There is a mailing list for Attic on librelist_ you can use for feature requests and general discussions about Attic. A mailing list archive is available `here `_. To subscribe to the list send an email to attic@librelist.com and reply to the confirmation mail. Likevise To unsubscribe send an email to attic-unsubscribe@librelist.com and reply to the confirmation mail. Attic-0.10/docs/installation.rst0000644000175000017500000000203412271172736017136 0ustar jonasjonas00000000000000.. include:: global.rst.inc .. _installation: Installation ============ |project_name| requires Python_ 3.2 or above to work. Even though Python 3 is not the default Python version on most Linux distributions it is usually available as an optional install. Other dependencies: * `msgpack-python`_ >= 0.1.10 * OpenSSL_ >= 1.0.0 The OpenSSL version bundled with Mac OS X and FreeBSD is most likey too old. Newer versions are available from homebrew_ on OS X and from FreeBSD ports. Installing from PyPI using pip ------------------------------ :: $ pip install Attic Installing from source tarballs ------------------------------- .. parsed-literal:: $ curl -O |package_url| $ tar -xvzf |package_filename| $ cd |package_dirname| $ python setup.py install Installing from git ------------------- .. parsed-literal:: $ git clone |git_url| $ cd attic $ python setup.py install Please not that when installing from git Cython_ is required to generate some files that are normally bundled with the release tarball. Attic-0.10/docs/quickstart.rst0000644000175000017500000000770312201414504016620 0ustar jonasjonas00000000000000.. include:: global.rst.inc .. _quickstart: Quick Start =========== This chapter will get you started with |project_name|. The first section presents a simple step by step example that uses |project_name| to backup data. The next section continues by showing how backups can be automated. A step by step example ---------------------- 1. Before any backup can be taken a repository has to be initialized:: $ attic init /somewhere/my-backup.attic 2. Backup the ``~/src`` and ``~/Documents`` directories into an archive called *first-backup*:: $ attic create -v /somwhere/my-backup.attic::first-backup ~/src ~/Documents 3. The next day create a new archive called *second-backup*:: $ attic create -v --stats /somwhere/my-backup.attic::second-backup ~/src ~/Documents This backup will be a lot quicker and a lot smaller since only new never before seen data is stored. The ``--stats`` causes |project_name| to output statistics about the newly created archive such as the amount of unique data (not shared with other archives). 4. List all archives in the repository:: $ attic list /somewhere/my-backup.attic 5. List the contents of the *first-backup* archive:: $ attic list /somewhere/my-backup.attic::first-backup 6. Restore the *first-backup* archive:: $ attic extract -v /somwhere/my-backup.attic::first-backup 7. Recover disk space by manually deleting the *first-backup* archive:: $ attic delete /somwhere/my-backup.attic::first-backup Automating backups ------------------ The following example script backups up ``/home`` and ``/var/www`` to a remote server. The script also uses the :ref:`attic_prune` subcommand to maintain a certain number of old archives:: #!/bin/sh REPOSITORY=username@remoteserver.com:backup.attic # Backup all of /home and /var/www except a few # excluded directories attic create --stats \ $REPOSITORY::hostname-`date +%Y-%m-%d` \ /home \ /var/www \ --exclude /home/*/.cache \ --exclude /home/Ben/Music/Justin\ Bieber \ --exclude *.pyc # Use the `prune` subcommand to maintain 7 daily, 4 weekly # and 6 monthly archives. attic prune -v $REPOSITORY --daily=7 --weekly=4 --monthly=6 .. Note:: This script assumes the repository has already been initalized with :ref:`attic_init`. .. _encrypted_repos: Repository encryption --------------------- Repository encryption is enabled at repository encryption time:: $ attic init --encryption=passphrase|keyfile PATH When repository encryption is enabled all data is encrypted using 256-bit AES_ encryption and the integrity and authenticity is verified using `HMAC-SHA256`_. |project_name| supports two different methods to derive the AES and HMAC keys. Passphrase based encryption This method uses a user supplied passphrase to derive the keys using the PBKDF2_ key derivation function. This method is convenient to use and secure as long as a *strong* passphrase is used. .. Note:: For automated backups the passphrase can be specified using the `ATTIC_PASSPHRASE` environment variable. Key file based encryption This method generates random keys at repository initialization time that are stored in a password protected file in the ``~/.attic/keys/`` directory. This method is secure and suitable for automated backups. .. Note:: The repository data is totally inaccessible without the key file so it must be kept **safe**. .. _remote_repos: Remote repositories ------------------- |project_name| can initialize and access repositories on remote hosts as the host is accessible using SSH and |project_name| is installed. The following syntax is used to address remote repositories:: $ attic init user@hostname:repository.attic or:: $ attic init ssh://user@hostname:port/repository.attic Attic-0.10/docs/update_usage.sh0000755000175000017500000000065312177542626016721 0ustar jonasjonas00000000000000#!/usr/bin/bash if [ ! -d usage ]; then mkdir usage fi for cmd in change-passphrase create delete extract info init list mount prune verify; do FILENAME="usage/$cmd.rst.inc" LINE=`echo -n attic $cmd | tr 'a-z- ' '-'` echo -e ".. _attic_$cmd:\n" > $FILENAME echo -e "attic $cmd\n$LINE\n::\n\n" >> $FILENAME attic $cmd -h | sed -e 's/^/ /' >> $FILENAME echo -e "\nDescription\n~~~~~~~~~~~\n\n" >> $FILENAME done Attic-0.10/docs/usage.rst0000644000175000017500000001421512223337170015535 0ustar jonasjonas00000000000000.. include:: global.rst.inc .. _detailed_usage: Usage ===== |project_name| consists of a number of commands. Each command accepts a number of arguments and options. The following sections will describe each command in detail. Quiet by default ---------------- Like most UNIX commands |project_name| is quiet by default but the ``-v`` or ``--verbose`` option can be used to get the program to output more status messages as it is processing. .. include:: usage/init.rst.inc This command initializes an empty :ref:`repository `. A repository is a filesystem directory containing the deduplicated data from zero or more archives. Encryption is enabled at repository initialization time. Examples ~~~~~~~~ :: # Local backup repository $ attic init /data/mybackuprepo.attic # Remote backup repository $ attic init user@hostname:mybackuprepo.attic # Encrypted remote backup repository $ attic init --encryption=passphrase user@hostname:mybackuprepo.attic .. include:: usage/create.rst.inc This command creates a backup archive containing all files found while recursively traversing all paths specified. The archive will consume almost no disk space for files or parts of files that has already been archived by other archives. Examples ~~~~~~~~ :: # Backups ~/Documents into an archive named "my-documents" $ attic create /data/myrepo.attic::my-documents ~/Documents # Backup ~/Documents and ~/src but exclude pyc files $ attic create /data/myrepo.attic::my-files \ ~/Documents \ ~/src \ --exclude *.pyc # Backup the root filesystem into an archive named "root-YYYY-MM-DD" NAME="root-`date +%Y-%m-%d`" $ attic create /data/myrepo.attic::$NAME / --do-not-cross-mountpoints .. include:: usage/extract.rst.inc This command extracts the contents of an archive. By default the entire archive is extracted but a subset of files and directories can be selected by passing a list of ``PATHs`` as arguments. The file selection can further be restricted by using the ``--exclude`` option. Examples ~~~~~~~~ :: # Extract entire archive $ attic extract /data/myrepo::my-files # Extract entire archive and list files while processing $ attic extract -v /data/myrepo::my-files # Extract the "src" directory $ attic extract /data/myrepo::my-files home/USERNAME/src # Extract the "src" directory but exclude object files $ attic extract /data/myrepo::my-files home/USERNAME/src --exclude *.o .. include:: usage/verify.rst.inc This command is similar to :ref:`attic_extract` but instead of writing any files to disk the command just verifies that all files are extractable and not corrupt. |project_name| will not compare the the archived files with the files on disk. .. include:: usage/delete.rst.inc This command deletes an archive from the repository. Any disk space not shared with any other existing archive is also reclaimed. .. include:: usage/list.rst.inc This command lists the contents of a repository or an archive. Examples ~~~~~~~~ :: $ attic list /data/myrepo my-files Thu Aug 1 23:33:22 2013 my-documents Thu Aug 1 23:35:43 2013 root-2013-08-01 Thu Aug 1 23:43:55 2013 root-2013-08-02 Fri Aug 2 15:18:17 2013 ... $ attic list /data/myrepo::root-2013-08-02 drwxr-xr-x root root 0 Jun 05 12:06 . lrwxrwxrwx root root 0 May 31 20:40 bin -> usr/bin drwxr-xr-x root root 0 Aug 01 22:08 etc drwxr-xr-x root root 0 Jul 15 22:07 etc/ImageMagick-6 -rw-r--r-- root root 1383 May 22 22:25 etc/ImageMagick-6/colors.xml ... .. include:: usage/prune.rst.inc The ``prune`` command prunes a repository by deleting archives not matching any of the specified retention options specified. This command is normally used by automated backup scripts wanting to keep a certain number of historic backups. Examples ~~~~~~~~ :: # Keep 7 end of day and 4 end of week archives $ attic prune /data/myrepo --daily=7 --weekly=4 # Same as above but only apply to archive names starting with "foo" $ attic prune /data/myrepo --daily=7 --weekly=4 --prefix=foo .. include:: usage/info.rst.inc This command displays some detailed information about the specified archive. Examples ~~~~~~~~ :: $ attic info /data/myrepo::root-2013-08-02 Name: root-2013-08-02 Fingerprint: bc3902e2c79b6d25f5d769b335c5c49331e6537f324d8d3badcb9a0917536dbb Hostname: myhostname Username: root Time: Fri Aug 2 15:18:17 2013 Command line: /usr/bin/attic create --stats /data/myrepo::root-2013-08-02 / --do-not-cross-mountpoints Number of files: 147429 Original size: 5344169493 (4.98 GB) Compressed size: 1748189642 (1.63 GB) Unique data: 64805454 (61.80 MB) .. include:: usage/mount.rst.inc This command mounts an archive as a FUSE filesystem. This can be useful for browsing an archive or restoring individual files. Unless the ``--foreground`` option is given the command will run in the background until the filesystem is ``umounted``. Examples ~~~~~~~~ :: $ attic mount /data/myrepo::root-2013-08-02 /tmp/mymountpoint $ ls /tmp/mymountpoint bin boot etc lib lib64 mnt opt root sbin srv usr var $ fusermount -u /tmp/mymountpoint .. include:: usage/change-passphrase.rst.inc The key files used for repository encryption are optionally passphrase protected. This command can be used to change this passphrase. Examples ~~~~~~~~ :: # Create a key file protected repository $ attic init --key-file /tmp/encrypted-repo Initializing repository at "/tmp/encrypted-repo" Enter passphrase (empty for no passphrase): Enter same passphrase again: Key file "/home/USER/.attic/keys/tmp_encrypted_repo" created. Keep this file safe. Your data will be inaccessible without it. # Change key file passphrase $ attic change-passphrase /tmp/encrypted-repo Enter passphrase for key file /home/USER/.attic/keys/tmp_encrypted_repo: New passphrase: Enter same passphrase again: Key file "/home/USER/.attic/keys/tmp_encrypted_repo" updated Attic-0.10/scripts/0000755000175000017500000000000012272541113014431 5ustar jonasjonas00000000000000Attic-0.10/scripts/attic0000644000175000017500000000007612166626110015466 0ustar jonasjonas00000000000000#!/usr/bin/env python from attic.archiver import main main() Attic-0.10/CHANGES0000644000175000017500000000340712272540220013737 0ustar jonasjonas00000000000000Attic Changelog =============== Here you can see the full list of changes between each Attic release. Version 0.10 ------------ (bugfix release, released on Jan 30, 2014) - Fix deadlock when extracting 0 sized files from remote repositories - "--exclude" wildcard patterns are now properly applied to the full path not just the file name part (#5). - Make source code endianness agnostic (#1) Version 0.9 ----------- (feature release, released on Jan 23, 2014) - Remote repository speed and reliability improvements. - Fix sorting of segment names to ignore NFS left over files. (#17) - Fix incorrect display of time (#13) - Improved error handling / reporting. (#12) - Use fcntl() instead of flock() when locking repository/cache. (#15) - Let ssh figure out port/user if not specified so we don't override .ssh/config (#9) - Improved libcrypto path detection (#23). Version 0.8.1 ------------- (bugfix release, released on Oct 4, 2013) - Fix segmentation fault issue. Version 0.8 ----------- (feature release, released on Oct 3, 2013) - Fix xattr issue when backing up sshfs filesystems (#4) - Fix issue with excessive index file size (#6) - Support access of read only repositories. - New syntax to enable repository encryption: attic init --encryption="none|passphrase|keyfile". - Detect and abort if repository is older than the cache. Version 0.7 ----------- (feature release, released on Aug 5, 2013) - Ported to FreeBSD - Improved documentation - Experimental: Archives mountable as fuse filesystems. - The "user." prefix is no longer stripped from xattrs on Linux Version 0.6.1 ------------- (bugfix release, released on July 19, 2013) - Fixed an issue where mtime was not always correctly restored. Version 0.6 ----------- First public release on July 9, 2013 Attic-0.10/LICENSE0000644000175000017500000000263512261537600013761 0ustar jonasjonas00000000000000Copyright (C) 2010-2014 Jonas Borgström All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Attic-0.10/MANIFEST.in0000644000175000017500000000024212200007327014471 0ustar jonasjonas00000000000000include README.rst LICENSE CHANGES MANIFEST.in versioneer.py recursive-include docs * recursive-exclude docs *.pyc recursive-exclude docs *.pyo prune docs/_build Attic-0.10/README.rst0000644000175000017500000000337112200252654014435 0ustar jonasjonas00000000000000What is Attic? -------------- Attic is a deduplicating backup program. The main goal of attic is to provide an efficient and secure way to backup data. The data deduplication technique used makes Attic suitable for daily backups since only actual changes are stored. Easy to use ~~~~~~~~~~~ Initialze backup repository and create a backup archive:: $ attic init /usbdrive/my-backup.attic $ attic create -v /usbdrive/my-backup.attic::documents ~/Documents Main features ~~~~~~~~~~~~~ Space efficient storage Variable block size deduplication is used to reduce the number of bytes stored by detecting redundant data. Each file is split into a number of variable length chunks and only chunks that have never been seen before are compressed and added to the repository. Optional data encryption All data can be protected using 256-bit AES encryption and data integrity and authenticity is verified using HMAC-SHA256. Off-site backups attic can store data on any remote host accessible over SSH as long as attic is installed. Backups mountable as filesystems Backup archives are mountable as userspace filesystems for easy backup verification and restores. What do I need? --------------- Attic requires Python 3.2 or above to work. Besides Python attic also requires msgpack-python and sufficiently recent OpenSSL (>= 1.0.0). How do I install it? -------------------- :: $ pip install Attic Where are the docs? ------------------- Go to https://pythonhosted.org/Attic/ for a prebuilt version of the docs. You can also build them yourself form the docs folder. Where are the tests? -------------------- The tests are in the attic/testsuite package. To run the test suite use the following command:: $ python -m attic.testsuite.run Attic-0.10/setup.py0000644000175000017500000000574712200011567014465 0ustar jonasjonas00000000000000# -*- encoding: utf-8 *-* import os import sys from glob import glob import attic import versioneer versioneer.versionfile_source = 'attic/_version.py' versioneer.versionfile_build = 'attic/_version.py' versioneer.tag_prefix = '' versioneer.parentdir_prefix = 'Attic-' # dirname like 'myproject-1.2.0' min_python = (3, 2) if sys.version_info < min_python: print("Attic requires Python %d.%d or later" % min_python) sys.exit(1) try: from setuptools import setup, Extension except ImportError: from distutils.core import setup, Extension chunker_source = 'attic/chunker.pyx' hashindex_source = 'attic/hashindex.pyx' try: from Cython.Distutils import build_ext import Cython.Compiler.Main as cython_compiler class Sdist(versioneer.cmd_sdist): def __init__(self, *args, **kwargs): for src in glob('attic/*.pyx'): cython_compiler.compile(glob('attic/*.pyx'), cython_compiler.default_options) versioneer.cmd_sdist.__init__(self, *args, **kwargs) def make_distribution(self): self.filelist.extend(['attic/chunker.c', 'attic/_chunker.c', 'attic/hashindex.c', 'attic/_hashindex.c']) super(Sdist, self).make_distribution() except ImportError: class Sdist(versioneer.cmd_sdist): def __init__(self, *args, **kwargs): raise Exception('Cython is required to run sdist') chunker_source = chunker_source.replace('.pyx', '.c') hashindex_source = hashindex_source.replace('.pyx', '.c') from distutils.command.build_ext import build_ext if not os.path.exists(chunker_source) or not os.path.exists(hashindex_source): raise ImportError('The GIT version of attic needs Cython. Install Cython or use a released version') with open('README.rst', 'r') as fd: long_description = fd.read() cmdclass = versioneer.get_cmdclass() cmdclass.update({'build_ext': build_ext, 'sdist': Sdist}) setup( name='Attic', version=versioneer.get_version(), author='Jonas Borgström', author_email='jonas@borgstrom.se', url='https://pythonhosted.org/Attic/', description='Deduplicated backups', long_description=long_description, license='BSD', platforms=['Linux', 'MacOS X'], classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Console', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', 'Operating System :: POSIX :: BSD :: FreeBSD', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Topic :: Security :: Cryptography', 'Topic :: System :: Archiving :: Backup', ], packages=['attic', 'attic.testsuite'], scripts=['scripts/attic'], cmdclass=cmdclass, ext_modules=[ Extension('attic.chunker', [chunker_source]), Extension('attic.hashindex', [hashindex_source]) ], install_requires=['msgpack-python'] ) Attic-0.10/versioneer.py0000644000175000017500000006222412200011340015464 0ustar jonasjonas00000000000000"""versioneer.py (like a rocketeer, but for versions) * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain * Version: 0.7+ This file helps distutils-based projects manage their version number by just creating version-control tags. For developers who work from a VCS-generated tree (e.g. 'git clone' etc), each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a version number by asking your version-control tool about the current checkout. The version number will be written into a generated _version.py file of your choosing, where it can be included by your __init__.py For users who work from a VCS-generated tarball (e.g. 'git archive'), it will compute a version number by looking at the name of the directory created when te tarball is unpacked. This conventionally includes both the name of the project and a version number. For users who work from a tarball built by 'setup.py sdist', it will get a version number from a previously-generated _version.py file. As a result, loading code directly from the source tree will not result in a real version. If you want real versions from VCS trees (where you frequently update from the upstream repository, or do new development), you will need to do a 'setup.py version' after each update, and load code from the build/ directory. You need to provide this code with a few configuration values: versionfile_source: A project-relative pathname into which the generated version strings should be written. This is usually a _version.py next to your project's main __init__.py file. If your project uses src/myproject/__init__.py, this should be 'src/myproject/_version.py'. This file should be checked in to your VCS as usual: the copy created below by 'setup.py update_files' will include code that parses expanded VCS keywords in generated tarballs. The 'build' and 'sdist' commands will replace it with a copy that has just the calculated version string. versionfile_build: Like versionfile_source, but relative to the build directory instead of the source directory. These will differ when your setup.py uses 'package_dir='. If you have package_dir={'myproject': 'src/myproject'}, then you will probably have versionfile_build='myproject/_version.py' and versionfile_source='src/myproject/_version.py'. tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. If your tags look like 'myproject-1.2.0', then you should use tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this should be an empty string. parentdir_prefix: a string, frequently the same as tag_prefix, which appears at the start of all unpacked tarball filenames. If your tarball unpacks into 'myproject-1.2.0', this should be 'myproject-'. To use it: 1: include this file in the top level of your project 2: make the following changes to the top of your setup.py: import versioneer versioneer.versionfile_source = 'src/myproject/_version.py' versioneer.versionfile_build = 'myproject/_version.py' versioneer.tag_prefix = '' # tags are like 1.2.0 versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0' 3: add the following arguments to the setup() call in your setup.py: version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), 4: run 'setup.py update_files', which will create _version.py, and will append the following to your __init__.py: from _version import __version__ 5: modify your MANIFEST.in to include versioneer.py 6: add both versioneer.py and the generated _version.py to your VCS """ import os, sys, re from distutils.core import Command from distutils.command.sdist import sdist as _sdist from distutils.command.build import build as _build versionfile_source = None versionfile_build = None tag_prefix = None parentdir_prefix = None VCS = "git" IN_LONG_VERSION_PY = False LONG_VERSION_PY = ''' IN_LONG_VERSION_PY = True # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (build by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by # versioneer-0.7+ (https://github.com/warner/python-versioneer) # these strings will be replaced by git during git-archive git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" import subprocess import sys def run_command(args, cwd=None, verbose=False): try: # remember shell=False, so use git.cmd on windows, not just git p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) except EnvironmentError: e = sys.exc_info()[1] if verbose: print("unable to run %%s" %% args[0]) print(e) return None stdout = p.communicate()[0].strip() if sys.version >= '3': stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %%s (error)" %% args[0]) return None return stdout import sys import re import os.path def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import # _version.py, so we do it with a regexp instead. This function is not # used from _version.py. variables = {} try: for line in open(versionfile_source,"r").readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: variables["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: variables["full"] = mo.group(1) except EnvironmentError: pass return variables def versions_from_expanded_variables(variables, tag_prefix, verbose=False): refnames = variables["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("variables are unexpanded, not using") return {} # unexpanded, so not in an unpacked git-archive tarball refs = set([r.strip() for r in refnames.strip("()").split(",")]) for ref in list(refs): if not re.search(r'\d', ref): if verbose: print("discarding '%%s', no digits" %% ref) refs.discard(ref) # Assume all version tags have a digit. git's %%d expansion # behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us # distinguish between branches and tags. By ignoring refnames # without digits, we filter out many common branch names like # "release" and "stabilization", as well as "HEAD" and "master". if verbose: print("remaining refs: %%s" %% ",".join(sorted(refs))) for ref in sorted(refs): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] if verbose: print("picking %%s" %% r) return { "version": r, "full": variables["full"].strip() } # no suitable tags, so we use the full revision id if verbose: print("no suitable tags, using full revision id") return { "version": variables["full"].strip(), "full": variables["full"].strip() } def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): # this runs 'git' from the root of the source tree. That either means # someone ran a setup.py command (and this code is in versioneer.py, so # IN_LONG_VERSION_PY=False, thus the containing directory is the root of # the source tree), or someone ran a project-specific entry point (and # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the # containing directory is somewhere deeper in the source tree). This only # gets called if the git-archive 'subst' variables were *not* expanded, # and _version.py hasn't already been rewritten with a short version # string, meaning we're inside a checked out source tree. try: here = os.path.abspath(__file__) except NameError: # some py2exe/bbfreeze/non-CPython implementations don't do __file__ return {} # not always correct # versionfile_source is the relative path from the top of the source tree # (where the .git directory might live) to this file. Invert this to find # the root from __file__. root = here if IN_LONG_VERSION_PY: for i in range(len(versionfile_source.split("/"))): root = os.path.dirname(root) else: root = os.path.dirname(here) if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %%s" %% root) return {} GIT = "git" if sys.platform == "win32": GIT = "git.cmd" stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: return {} if not stdout.startswith(tag_prefix): if verbose: print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix)) return {} tag = stdout[len(tag_prefix):] stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) if stdout is None: return {} full = stdout.strip() if tag.endswith("-dirty"): full += "-dirty" return {"version": tag, "full": full} def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): if IN_LONG_VERSION_PY: # We're running from _version.py. If it's from a source tree # (execute-in-place), we can work upwards to find the root of the # tree, and then check the parent directory for a version string. If # it's in an installed application, there's no hope. try: here = os.path.abspath(__file__) except NameError: # py2exe/bbfreeze/non-CPython don't have __file__ return {} # without __file__, we have no hope # versionfile_source is the relative path from the top of the source # tree to _version.py. Invert this to find the root from __file__. root = here for i in range(len(versionfile_source.split("/"))): root = os.path.dirname(root) else: # we're running from versioneer.py, which means we're running from # the setup.py in a source tree. sys.argv[0] is setup.py in the root. here = os.path.abspath(sys.argv[0]) root = os.path.dirname(here) # Source tarballs conventionally unpack into a directory that includes # both the project name and a version string. dirname = os.path.basename(root) if not dirname.startswith(parentdir_prefix): if verbose: print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %% (root, dirname, parentdir_prefix)) return None return {"version": dirname[len(parentdir_prefix):], "full": ""} tag_prefix = "%(TAG_PREFIX)s" parentdir_prefix = "%(PARENTDIR_PREFIX)s" versionfile_source = "%(VERSIONFILE_SOURCE)s" def get_versions(default={"version": "unknown", "full": ""}, verbose=False): variables = { "refnames": git_refnames, "full": git_full } ver = versions_from_expanded_variables(variables, tag_prefix, verbose) if not ver: ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) if not ver: ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose) if not ver: ver = default return ver ''' import subprocess import sys def run_command(args, cwd=None, verbose=False): try: # remember shell=False, so use git.cmd on windows, not just git p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) except EnvironmentError: e = sys.exc_info()[1] if verbose: print("unable to run %s" % args[0]) print(e) return None stdout = p.communicate()[0].strip() if sys.version >= '3': stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %s (error)" % args[0]) return None return stdout import sys import re import os.path def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import # _version.py, so we do it with a regexp instead. This function is not # used from _version.py. variables = {} try: for line in open(versionfile_source,"r").readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: variables["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: variables["full"] = mo.group(1) except EnvironmentError: pass return variables def versions_from_expanded_variables(variables, tag_prefix, verbose=False): refnames = variables["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("variables are unexpanded, not using") return {} # unexpanded, so not in an unpacked git-archive tarball refs = set([r.strip() for r in refnames.strip("()").split(",")]) for ref in list(refs): if not re.search(r'\d', ref): if verbose: print("discarding '%s', no digits" % ref) refs.discard(ref) # Assume all version tags have a digit. git's %d expansion # behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us # distinguish between branches and tags. By ignoring refnames # without digits, we filter out many common branch names like # "release" and "stabilization", as well as "HEAD" and "master". if verbose: print("remaining refs: %s" % ",".join(sorted(refs))) for ref in sorted(refs): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) return { "version": r, "full": variables["full"].strip() } # no suitable tags, so we use the full revision id if verbose: print("no suitable tags, using full revision id") return { "version": variables["full"].strip(), "full": variables["full"].strip() } def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): # this runs 'git' from the root of the source tree. That either means # someone ran a setup.py command (and this code is in versioneer.py, so # IN_LONG_VERSION_PY=False, thus the containing directory is the root of # the source tree), or someone ran a project-specific entry point (and # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the # containing directory is somewhere deeper in the source tree). This only # gets called if the git-archive 'subst' variables were *not* expanded, # and _version.py hasn't already been rewritten with a short version # string, meaning we're inside a checked out source tree. try: here = os.path.abspath(__file__) except NameError: # some py2exe/bbfreeze/non-CPython implementations don't do __file__ return {} # not always correct # versionfile_source is the relative path from the top of the source tree # (where the .git directory might live) to this file. Invert this to find # the root from __file__. root = here if IN_LONG_VERSION_PY: for i in range(len(versionfile_source.split("/"))): root = os.path.dirname(root) else: root = os.path.dirname(here) if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) return {} GIT = "git" if sys.platform == "win32": GIT = "git.cmd" stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: return {} if not stdout.startswith(tag_prefix): if verbose: print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) return {} tag = stdout[len(tag_prefix):] stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) if stdout is None: return {} full = stdout.strip() if tag.endswith("-dirty"): full += "-dirty" return {"version": tag, "full": full} def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): if IN_LONG_VERSION_PY: # We're running from _version.py. If it's from a source tree # (execute-in-place), we can work upwards to find the root of the # tree, and then check the parent directory for a version string. If # it's in an installed application, there's no hope. try: here = os.path.abspath(__file__) except NameError: # py2exe/bbfreeze/non-CPython don't have __file__ return {} # without __file__, we have no hope # versionfile_source is the relative path from the top of the source # tree to _version.py. Invert this to find the root from __file__. root = here for i in range(len(versionfile_source.split("/"))): root = os.path.dirname(root) else: # we're running from versioneer.py, which means we're running from # the setup.py in a source tree. sys.argv[0] is setup.py in the root. here = os.path.abspath(sys.argv[0]) root = os.path.dirname(here) # Source tarballs conventionally unpack into a directory that includes # both the project name and a version string. dirname = os.path.basename(root) if not dirname.startswith(parentdir_prefix): if verbose: print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % (root, dirname, parentdir_prefix)) return None return {"version": dirname[len(parentdir_prefix):], "full": ""} import sys def do_vcs_install(versionfile_source, ipy): GIT = "git" if sys.platform == "win32": GIT = "git.cmd" run_command([GIT, "add", "versioneer.py"]) run_command([GIT, "add", versionfile_source]) run_command([GIT, "add", ipy]) present = False try: f = open(".gitattributes", "r") for line in f.readlines(): if line.strip().startswith(versionfile_source): if "export-subst" in line.strip().split()[1:]: present = True f.close() except EnvironmentError: pass if not present: f = open(".gitattributes", "a+") f.write("%s export-subst\n" % versionfile_source) f.close() run_command([GIT, "add", ".gitattributes"]) SHORT_VERSION_PY = """ # This file was generated by 'versioneer.py' (0.7+) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. version_version = '%(version)s' version_full = '%(full)s' def get_versions(default={}, verbose=False): return {'version': version_version, 'full': version_full} """ DEFAULT = {"version": "unknown", "full": "unknown"} def versions_from_file(filename): versions = {} try: f = open(filename) except EnvironmentError: return versions for line in f.readlines(): mo = re.match("version_version = '([^']+)'", line) if mo: versions["version"] = mo.group(1) mo = re.match("version_full = '([^']+)'", line) if mo: versions["full"] = mo.group(1) return versions def write_to_version_file(filename, versions): f = open(filename, "w") f.write(SHORT_VERSION_PY % versions) f.close() print("set %s to '%s'" % (filename, versions["version"])) def get_best_versions(versionfile, tag_prefix, parentdir_prefix, default=DEFAULT, verbose=False): # returns dict with two keys: 'version' and 'full' # # extract version from first of _version.py, 'git describe', parentdir. # This is meant to work for developers using a source checkout, for users # of a tarball created by 'setup.py sdist', and for users of a # tarball/zipball created by 'git archive' or github's download-from-tag # feature. variables = get_expanded_variables(versionfile_source) if variables: ver = versions_from_expanded_variables(variables, tag_prefix) if ver: if verbose: print("got version from expanded variable %s" % ver) return ver ver = versions_from_file(versionfile) if ver: if verbose: print("got version from file %s %s" % (versionfile, ver)) return ver ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) if ver: if verbose: print("got version from git %s" % ver) return ver ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose) if ver: if verbose: print("got version from parentdir %s" % ver) return ver if verbose: print("got version from default %s" % ver) return default def get_versions(default=DEFAULT, verbose=False): assert versionfile_source is not None, "please set versioneer.versionfile_source" assert tag_prefix is not None, "please set versioneer.tag_prefix" assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix" return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix, default=default, verbose=verbose) def get_version(verbose=False): return get_versions(verbose=verbose)["version"] class cmd_version(Command): description = "report generated version string" user_options = [] boolean_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): ver = get_version(verbose=True) print("Version is currently: %s" % ver) class cmd_build(_build): def run2(self): versions = get_versions(verbose=True) _build.run(self) # now locate _version.py in the new build/ directory and replace it # with an updated value target_versionfile = os.path.join(self.build_lib, versionfile_build) print("UPDATING %s" % target_versionfile) os.unlink(target_versionfile) f = open(target_versionfile, "w") f.write(SHORT_VERSION_PY % versions) f.close() class cmd_sdist(_sdist): def run(self): versions = get_versions(verbose=True) self._versioneer_generated_versions = versions # unless we update this, the command will keep using the old version self.distribution.metadata.version = versions["version"] return _sdist.run(self) def make_release_tree(self, base_dir, files): _sdist.make_release_tree(self, base_dir, files) # now locate _version.py in the new base_dir directory (remembering # that it may be a hardlink) and replace it with an updated value target_versionfile = os.path.join(base_dir, versionfile_source) print("UPDATING %s" % target_versionfile) os.unlink(target_versionfile) f = open(target_versionfile, "w") f.write(SHORT_VERSION_PY % self._versioneer_generated_versions) f.close() INIT_PY_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ class cmd_update_files(Command): description = "modify __init__.py and create _version.py" user_options = [] boolean_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py") print(" creating %s" % versionfile_source) f = open(versionfile_source, "w") f.write(LONG_VERSION_PY % {"DOLLAR": "$", "TAG_PREFIX": tag_prefix, "PARENTDIR_PREFIX": parentdir_prefix, "VERSIONFILE_SOURCE": versionfile_source, }) f.close() try: old = open(ipy, "r").read() except EnvironmentError: old = "" if INIT_PY_SNIPPET not in old: print(" appending to %s" % ipy) f = open(ipy, "a") f.write(INIT_PY_SNIPPET) f.close() else: print(" %s unmodified" % ipy) do_vcs_install(versionfile_source, ipy) def get_cmdclass(): return {'version': cmd_version, 'update_files': cmd_update_files, 'build': cmd_build, 'sdist': cmd_sdist, } Attic-0.10/PKG-INFO0000644000175000017500000000560512272541113014045 0ustar jonasjonas00000000000000Metadata-Version: 1.1 Name: Attic Version: 0.10 Summary: Deduplicated backups Home-page: https://pythonhosted.org/Attic/ Author: Jonas Borgström Author-email: jonas@borgstrom.se License: BSD Description: What is Attic? -------------- Attic is a deduplicating backup program. The main goal of attic is to provide an efficient and secure way to backup data. The data deduplication technique used makes Attic suitable for daily backups since only actual changes are stored. Easy to use ~~~~~~~~~~~ Initialze backup repository and create a backup archive:: $ attic init /usbdrive/my-backup.attic $ attic create -v /usbdrive/my-backup.attic::documents ~/Documents Main features ~~~~~~~~~~~~~ Space efficient storage Variable block size deduplication is used to reduce the number of bytes stored by detecting redundant data. Each file is split into a number of variable length chunks and only chunks that have never been seen before are compressed and added to the repository. Optional data encryption All data can be protected using 256-bit AES encryption and data integrity and authenticity is verified using HMAC-SHA256. Off-site backups attic can store data on any remote host accessible over SSH as long as attic is installed. Backups mountable as filesystems Backup archives are mountable as userspace filesystems for easy backup verification and restores. What do I need? --------------- Attic requires Python 3.2 or above to work. Besides Python attic also requires msgpack-python and sufficiently recent OpenSSL (>= 1.0.0). How do I install it? -------------------- :: $ pip install Attic Where are the docs? ------------------- Go to https://pythonhosted.org/Attic/ for a prebuilt version of the docs. You can also build them yourself form the docs folder. Where are the tests? -------------------- The tests are in the attic/testsuite package. To run the test suite use the following command:: $ python -m attic.testsuite.run Platform: Linux Platform: MacOS X Classifier: Development Status :: 4 - Beta Classifier: Environment :: Console Classifier: Intended Audience :: System Administrators Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: POSIX :: BSD :: FreeBSD Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX :: Linux Classifier: Programming Language :: Python Classifier: Topic :: Security :: Cryptography Classifier: Topic :: System :: Archiving :: Backup