pax_global_header00006660000000000000000000000064141507200220014503gustar00rootroot0000000000000052 comment=732c3b28dbef305cd26b91d608a1a60caf09e588 critnib-1.1/000077500000000000000000000000001415072002200127765ustar00rootroot00000000000000critnib-1.1/.cirrus.yml000066400000000000000000000012051415072002200151040ustar00rootroot00000000000000freebsd_13_task: freebsd_instance: image: freebsd-13-0-release-amd64 install_script: pkg install -y cmake perl5 build_script: - NPROC=$(getconf _NPROCESSORS_ONLN) - cmake . - make -j${NPROC} test_script: # run tests as user "cirrus" instead of root - pw useradd cirrus -m - chown -R cirrus:cirrus . - sudo -u cirrus ctest --output-on-failure macos_x_task: macos_instance: image: big-sur-xcode install_script: - brew update - brew install cmake build_script: - NPROC=$(getconf _NPROCESSORS_ONLN) - cmake . - make -j${NPROC} test_script: ctest --output-on-failure critnib-1.1/.gitignore000066400000000000000000000001541415072002200147660ustar00rootroot00000000000000*~ *.o *.so *.so.* *.3 m CMakeCache.txt CMakeFiles Testing CTestTestfile.cmake cmake_install.cmake Makefile critnib-1.1/CMakeLists.txt000066400000000000000000000022331415072002200155360ustar00rootroot00000000000000cmake_minimum_required(VERSION 2.8.12) project(critnib C) set(THREADS_PREFER_PTHREAD_FLAG TRUE) include(FindThreads) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") endif (NOT CMAKE_BUILD_TYPE) set(CMAKE_C_FLAGS "-Wall -Werror=format-security ${CMAKE_C_FLAGS}") add_library(critnib SHARED critnib.c) set(libs critnib ${CMAKE_THREAD_LIBS_INIT} -latomic) add_executable(m main.c) target_link_libraries(m ${libs}) set_target_properties(critnib PROPERTIES SOVERSION 1 PUBLIC_HEADER "critnib.h") include(GNUInstallDirs) install(TARGETS critnib PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/) install(FILES critnib.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libcritnib.3 critnib.3 DESTINATION ${CMAKE_INSTALL_MANDIR}/man3) enable_testing() add_subdirectory(tests) add_custom_command(OUTPUT libcritnib.3 DEPENDS libcritnib.pod COMMAND pod2man -c critnib -s 3 -n libcritnib <${CMAKE_CURRENT_SOURCE_DIR}/libcritnib.pod >${CMAKE_CURRENT_BINARY_DIR}/libcritnib.3) add_custom_target(man ALL DEPENDS libcritnib.3) critnib-1.1/README.md000066400000000000000000000020511415072002200142530ustar00rootroot00000000000000critnib ======= This library provides a lock-free implementation of *critnib* — a cross between radix trees and D.J. Bernstein's critbit. The structure offers searches for equality and for next neighbour. This implementation is restricted to fixed sized keys — an intptr_t in this library — even though the abstract data structure can support arbitrary key lengths. Lock-free mutations haven't been implemented either, although they're relatively easy to add. On the other hand, reads are strongly optimized, and do not require any kind of bus synchronizations (which are very costly on a many-core multi-socket machine). That is, you will want this library if you need reads inside tight performance-critical loops but writes don't require this level of optimization. Copyright ========= This code has been developed at Intel as an internal part of PMDK, then extracted by the author and adapted for use as a stand-alone library. Thus, while the copyright and praise belong to Intel, this library is not a product supported or validated by Intel. critnib-1.1/critnib.3000066400000000000000000000000211415072002200145050ustar00rootroot00000000000000.so libcritnib.3 critnib-1.1/critnib.c000066400000000000000000000436471415072002200146120ustar00rootroot00000000000000// SPDX-License-Identifier: BSD-3-Clause /* Copyright 2018-2019, Intel Corporation */ /* * critnib.c -- implementation of critnib tree * * It offers identity lookup (like a hashmap) and <= lookup (like a search * tree). Unlike some hashing algorithms (cuckoo hash, perfect hashing) the * complexity isn't constant, but for data sizes we expect it's several * times as fast as cuckoo, and has no "stop the world" cases that would * cause latency (ie, better worst case behaviour). */ /* * STRUCTURE DESCRIPTION * * Critnib is a hybrid between a radix tree and DJ Bernstein's critbit: * it skips nodes for uninteresting radix nodes (ie, ones that would have * exactly one child), this requires adding to every node a field that * describes the slice (4-bit in our case) that this radix level is for. * * This implementation also stores each node's path (ie, bits that are * common to every key in that subtree) -- this doesn't help with lookups * at all (unused in == match, could be reconstructed at no cost in <= * after first dive) but simplifies inserts and removes. If we ever want * that piece of memory it's easy to trim it down. */ /* * CONCURRENCY ISSUES * * Reads are completely lock-free sync-free, but only almost wait-free: * if for some reason a read thread gets pathologically stalled, it will * notice the data being stale and restart the work. In usual cases, * the structure having been modified does _not_ cause a restart. * * Writes could be easily made lock-free as well (with only a cmpxchg * sync), but this leads to problems with removes. A possible solution * would be doing removes by overwriting by NULL w/o freeing -- yet this * would lead to the structure growing without bounds. Complex per-node * locks would increase concurrency but they slow down individual writes * enough that in practice a simple global write lock works faster. * * Removes are the only operation that can break reads. The structure * can do local RCU well -- the problem being knowing when it's safe to * free. Any synchronization with reads would kill their speed, thus * instead we have a remove count. The grace period is DELETED_LIFE, * after which any read will notice staleness and restart its work. */ #include #include #include #include "critnib.h" #include "pmdk-compat.h" /* * A node that has been deleted is left untouched for this many delete * cycles. Reads have guaranteed correctness if they took no longer than * DELETED_LIFE concurrent deletes, otherwise they notice something is * wrong and restart. The memory of deleted nodes is never freed to * malloc nor their pointers lead anywhere wrong, thus a stale read will * (temporarily) get a wrong answer but won't crash. * * There's no need to count writes as they never interfere with reads. * * Allowing stale reads (of arbitrarily old writes or of deletes less than * DELETED_LIFE old) might sound counterintuitive, but it doesn't affect * semantics in any way: the thread could have been stalled just after * returning from our code. Thus, the guarantee is: the result of get() or * find_le() is a value that was current at any point between the call * start and end. */ #define DELETED_LIFE 16 #define SLICE 4 #define NIB ((1UL << SLICE) - 1) #define SLNODES (1 << SLICE) typedef uintptr_t word; typedef unsigned char sh_t; struct critnib_node { /* * path is the part of a tree that's already traversed (be it through * explicit nodes or collapsed links) -- ie, any subtree below has all * those bits set to this value. * * nib is a 4-bit slice that's an index into the node's children. * * shift is the length (in bits) of the part of the key below this node. * * nib * |XXXXXXXXXX|?|*****| * path ^ * +-----+ * shift */ struct critnib_node *child[SLNODES]; word path; sh_t shift; }; struct critnib_leaf { word key; void *value; }; struct critnib { struct critnib_node *root; /* pool of freed nodes: singly linked list, next at child[0] */ struct critnib_node *deleted_node; struct critnib_leaf *deleted_leaf; /* nodes removed but not yet eligible for reuse */ struct critnib_node *pending_del_nodes[DELETED_LIFE]; struct critnib_leaf *pending_del_leaves[DELETED_LIFE]; uint64_t remove_count; os_mutex_t mutex; /* writes/removes */ }; /* * atomic load */ static void load(void *src, void *dst) { __atomic_load((word *)src, (word *)dst, memory_order_acquire); } static void load64(uint64_t *src, uint64_t *dst) { __atomic_load(src, dst, memory_order_acquire); } /* * atomic store */ static void store(void *dst, void *src) { __atomic_store_n((word *)dst, (word)src, memory_order_release); } /* * internal: is_leaf -- check tagged pointer for leafness */ static inline bool is_leaf(struct critnib_node *n) { return (word)n & 1; } /* * internal: to_leaf -- untag a leaf pointer */ static inline struct critnib_leaf * to_leaf(struct critnib_node *n) { return (void *)((word)n & ~1UL); } /* * internal: path_mask -- return bit mask of a path above a subtree [shift] * bits tall */ static inline word path_mask(sh_t shift) { return ~NIB << shift; } /* * internal: slice_index -- return index of child at the given nib */ static inline unsigned slice_index(word key, sh_t shift) { return (unsigned)((key >> shift) & NIB); } /* * critnib_new -- allocates a new critnib structure */ struct critnib * critnib_new(void) { struct critnib *c = Zalloc(sizeof(struct critnib)); if (!c) return NULL; util_mutex_init(&c->mutex); VALGRIND_HG_DRD_DISABLE_CHECKING(&c->root, sizeof(c->root)); VALGRIND_HG_DRD_DISABLE_CHECKING(&c->remove_count, sizeof(c->remove_count)); return c; } /* * internal: delete_node -- recursively free (to malloc) a subtree */ static void delete_node(struct critnib_node *__restrict n) { if (!is_leaf(n)) { for (int i = 0; i < SLNODES; i++) { if (n->child[i]) delete_node(n->child[i]); } Free(n); } else { Free(to_leaf(n)); } } /* * critnib_delete -- destroy and free a critnib struct */ void critnib_delete(struct critnib *c) { if (c->root) delete_node(c->root); util_mutex_destroy(&c->mutex); for (struct critnib_node *m = c->deleted_node; m; ) { struct critnib_node *mm = m->child[0]; Free(m); m = mm; } for (struct critnib_leaf *k = c->deleted_leaf; k; ) { struct critnib_leaf *kk = k->value; Free(k); k = kk; } for (int i = 0; i < DELETED_LIFE; i++) { Free(c->pending_del_nodes[i]); Free(c->pending_del_leaves[i]); } Free(c); } /* * internal: free_node -- free (to internal pool, not malloc) a node. * * We cannot free them to malloc as a stalled reader thread may still walk * through such nodes; it will notice the result being bogus but only after * completing the walk, thus we need to ensure any freed nodes still point * to within the critnib structure. */ static void free_node(struct critnib *__restrict c, struct critnib_node *__restrict n) { if (!n) return; ASSERT(!is_leaf(n)); n->child[0] = c->deleted_node; c->deleted_node = n; } /* * internal: alloc_node -- allocate a node from our pool or from malloc */ static struct critnib_node * alloc_node(struct critnib *__restrict c) { if (!c->deleted_node) { struct critnib_node *n = Malloc(sizeof(struct critnib_node)); if (n == NULL) ERR("!Malloc"); return n; } struct critnib_node *n = c->deleted_node; c->deleted_node = n->child[0]; VALGRIND_ANNOTATE_NEW_MEMORY(n, sizeof(*n)); return n; } /* * internal: free_leaf -- free (to internal pool, not malloc) a leaf. * * See free_node(). */ static void free_leaf(struct critnib *__restrict c, struct critnib_leaf *__restrict k) { if (!k) return; k->value = c->deleted_leaf; c->deleted_leaf = k; } /* * internal: alloc_leaf -- allocate a leaf from our pool or from malloc */ static struct critnib_leaf * alloc_leaf(struct critnib *__restrict c) { if (!c->deleted_leaf) { struct critnib_leaf *k = Malloc(sizeof(struct critnib_leaf)); if (k == NULL) ERR("!Malloc"); return k; } struct critnib_leaf *k = c->deleted_leaf; c->deleted_leaf = k->value; VALGRIND_ANNOTATE_NEW_MEMORY(k, sizeof(*k)); return k; } /* * crinib_insert -- write a key:value pair to the critnib structure * * Returns: * • 0 on success * • EEXIST if such a key already exists * • ENOMEM if we're out of memory * * Takes a global write lock but doesn't stall any readers. */ int critnib_insert(struct critnib *c, word key, void *value, int update) { util_mutex_lock(&c->mutex); struct critnib_leaf *k = alloc_leaf(c); if (!k) { util_mutex_unlock(&c->mutex); return ENOMEM; } VALGRIND_HG_DRD_DISABLE_CHECKING(k, sizeof(struct critnib_leaf)); k->key = key; k->value = value; struct critnib_node *kn = (void *)((word)k | 1); struct critnib_node *n = c->root; if (!n) { c->root = kn; util_mutex_unlock(&c->mutex); return 0; } struct critnib_node **parent = &c->root; struct critnib_node *prev = c->root; while (n && !is_leaf(n) && (key & path_mask(n->shift)) == n->path) { prev = n; parent = &n->child[slice_index(key, n->shift)]; n = *parent; } if (!n) { n = prev; store(&n->child[slice_index(key, n->shift)], kn); util_mutex_unlock(&c->mutex); return 0; } word path = is_leaf(n) ? to_leaf(n)->key : n->path; /* Find where the path differs from our key. */ word at = path ^ key; if (!at) { ASSERT(is_leaf(n)); free_leaf(c, to_leaf(kn)); if (update) { to_leaf(n)->value = value; util_mutex_unlock(&c->mutex); return 0; } else { util_mutex_unlock(&c->mutex); return EEXIST; } } /* and convert that to an index. */ #if __SIZEOF_SIZE_T__ == 8 sh_t sh = util_mssb_index64(at) & (sh_t)~(SLICE - 1); #else sh_t sh = util_mssb_index32(at) & (sh_t)~(SLICE - 1); #endif struct critnib_node *m = alloc_node(c); if (!m) { free_leaf(c, to_leaf(kn)); util_mutex_unlock(&c->mutex); return ENOMEM; } VALGRIND_HG_DRD_DISABLE_CHECKING(m, sizeof(struct critnib_node)); for (int i = 0; i < SLNODES; i++) m->child[i] = NULL; m->child[slice_index(key, sh)] = kn; m->child[slice_index(path, sh)] = n; m->shift = sh; m->path = key & path_mask(sh); store(parent, m); util_mutex_unlock(&c->mutex); return 0; } /* * critnib_remove -- delete a key from the critnib structure, return its value */ void * critnib_remove(struct critnib *c, word key) { struct critnib_leaf *k; void *value = NULL; util_mutex_lock(&c->mutex); struct critnib_node *n = c->root; if (!n) goto not_found; word del = __atomic_fetch_add(&c->remove_count, 1, __ATOMIC_ACQ_REL) % DELETED_LIFE; free_node(c, c->pending_del_nodes[del]); free_leaf(c, c->pending_del_leaves[del]); c->pending_del_nodes[del] = NULL; c->pending_del_leaves[del] = NULL; if (is_leaf(n)) { k = to_leaf(n); if (k->key == key) { store(&c->root, NULL); goto del_leaf; } goto not_found; } /* * n and k are a parent:child pair (after the first iteration); k is the * leaf that holds the key we're deleting. */ struct critnib_node **k_parent = &c->root; struct critnib_node **n_parent = &c->root; struct critnib_node *kn = n; while (!is_leaf(kn)) { n_parent = k_parent; n = kn; k_parent = &kn->child[slice_index(key, kn->shift)]; kn = *k_parent; if (!kn) goto not_found; } k = to_leaf(kn); if (k->key != key) goto not_found; store(&n->child[slice_index(key, n->shift)], NULL); /* Remove the node if there's only one remaining child. */ int ochild = -1; for (int i = 0; i < SLNODES; i++) { if (n->child[i]) { if (ochild != -1) goto del_leaf; ochild = i; } } ASSERTne(ochild, -1); store(n_parent, n->child[ochild]); c->pending_del_nodes[del] = n; del_leaf: value = k->value; c->pending_del_leaves[del] = k; not_found: util_mutex_unlock(&c->mutex); return value; } /* * critnib_get -- query for a key ("==" match), returns value or NULL * * Doesn't need a lock but if many deletes happened while our thread was * somehow stalled the query is restarted (as freed nodes remain unused only * for a grace period). * * Counterintuitively, it's pointless to return the most current answer, * we need only one that was valid at any point after the call started. */ void * critnib_get(struct critnib *c, word key) { uint64_t wrs1, wrs2; void *res; do { struct critnib_node *n; load64(&c->remove_count, &wrs1); load(&c->root, &n); /* * critbit algorithm: dive into the tree, looking at nothing but * each node's critical bit^H^H^Hnibble. This means we risk * going wrong way if our path is missing, but that's ok... */ while (n && !is_leaf(n)) load(&n->child[slice_index(key, n->shift)], &n); /* ... as we check it at the end. */ struct critnib_leaf *k = to_leaf(n); res = (n && k->key == key) ? k->value : NULL; load64(&c->remove_count, &wrs2); } while (wrs1 + DELETED_LIFE <= wrs2); return res; } /* * internal: find_predecessor -- return the rightmost leaf in a subtree */ static struct critnib_leaf * find_predecessor(struct critnib_node *__restrict n) { while (1) { int nib; for (nib = NIB; nib >= 0; nib--) if (n->child[nib]) break; if (nib < 0) return NULL; n = n->child[nib]; if (is_leaf(n)) return to_leaf(n); } } /* * internal: find_le -- recursively search <= in a subtree */ static struct critnib_leaf * find_le(struct critnib_node *__restrict n, word key) { if (!n) return NULL; if (is_leaf(n)) { struct critnib_leaf *k = to_leaf(n); return (k->key <= key) ? k : NULL; } /* * is our key outside the subtree we're in? * * If we're inside, all bits above the nib will be identical; note * that shift points at the nib's lower rather than upper edge, so it * needs to be masked away as well. */ if ((key ^ n->path) >> (n->shift) & ~NIB) { /* * subtree is too far to the left? * -> its rightmost value is good */ if (n->path < key) return find_predecessor(n); /* * subtree is too far to the right? * -> it has nothing of interest to us */ return NULL; } unsigned nib = slice_index(key, n->shift); /* recursive call: follow the path */ { struct critnib_node *m; load(&n->child[nib], &m); struct critnib_leaf *k = find_le(m, key); if (k) return k; } /* * nothing in that subtree? We strayed from the path at this point, * thus need to search every subtree to our left in this node. No * need to dive into any but the first non-null, though. */ for (; nib > 0; nib--) { struct critnib_node *m; load(&n->child[nib - 1], &m); if (m) { n = m; if (is_leaf(n)) return to_leaf(n); return find_predecessor(n); } } return NULL; } /* * critnib_find_le -- query for a key ("<=" match), returns value or NULL * * Same guarantees as critnib_get(). */ void * critnib_find_le(struct critnib *c, word key) { uint64_t wrs1, wrs2; void *res; do { load64(&c->remove_count, &wrs1); struct critnib_node *n; /* avoid a subtle TOCTOU */ load(&c->root, &n); struct critnib_leaf *k = n ? find_le(n, key) : NULL; res = k ? k->value : NULL; load64(&c->remove_count, &wrs2); } while (wrs1 + DELETED_LIFE <= wrs2); return res; } /* * internal: find_successor -- return the rightmost leaf in a subtree */ static struct critnib_leaf * find_successor(struct critnib_node *__restrict n) { while (1) { int nib; for (nib = 0; nib <= NIB; nib++) if (n->child[nib]) break; if (nib > NIB) return NULL; n = n->child[nib]; if (is_leaf(n)) return to_leaf(n); } } /* * internal: find_ge -- recursively search >= in a subtree */ static struct critnib_leaf * find_ge(struct critnib_node *__restrict n, word key) { if (!n) return NULL; if (is_leaf(n)) { struct critnib_leaf *k = to_leaf(n); return (k->key >= key) ? k : NULL; } if ((key ^ n->path) >> (n->shift) & ~NIB) { if (n->path > key) return find_successor(n); return NULL; } unsigned nib = slice_index(key, n->shift); { struct critnib_node *m; load(&n->child[nib], &m); struct critnib_leaf *k = find_ge(m, key); if (k) return k; } for (; nib < NIB; nib++) { struct critnib_node *m; load(&n->child[nib + 1], &m); if (m) { n = m; if (is_leaf(n)) return to_leaf(n); return find_successor(n); } } return NULL; } /* * critnib_find -- parametrized query, returns 1 if found */ int critnib_find(struct critnib *c, uintptr_t key, enum find_dir_t dir, uintptr_t *rkey, void **rvalue) { uint64_t wrs1, wrs2; struct critnib_leaf *k; uintptr_t _rkey; void **_rvalue; /* <42 ≡ ≤41 */ if (dir < -1) { if (!key) return 0; key--; } else if (dir > +1) { if (key == -1) return 0; key++; } do { load64(&c->remove_count, &wrs1); struct critnib_node *n; load(&c->root, &n); if (dir < 0) k = find_le(n, key); else if (dir > 0) k = find_ge(n, key); else { while (n && !is_leaf(n)) load(&n->child[slice_index(key, n->shift)], &n); struct critnib_leaf *kk = to_leaf(n); k = (n && kk->key == key) ? kk : NULL; } if (k) { _rkey = k->key; _rvalue = k->value; } load64(&c->remove_count, &wrs2); } while (wrs1 + DELETED_LIFE <= wrs2); if (k) { if (rkey) *rkey = _rkey; if (rvalue) *rvalue = _rvalue; return 1; } return 0; } /* * critnib_iter -- iterator, [min..max], calls func(key, value, privdata) * * If func() returns non-zero, the search is aborted. */ static int iter(struct critnib_node *__restrict n, word min, word max, int (*func)(word key, void *value, void *privdata), void *privdata) { if (is_leaf(n)) { word k = to_leaf(n)->key; if (k >= min && k <= max) return func(to_leaf(n)->key, to_leaf(n)->value, privdata); return 0; } if (n->path > max) return 1; if ((n->path | path_mask(n->shift)) < min) return 0; for (int i = 0; i < SLNODES; i++) { struct critnib_node *__restrict m = n->child[i]; if (m && iter(m, min, max, func, privdata)) return 1; } return 0; } void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, int (*func)(uintptr_t key, void *value, void *privdata), void *privdata) { util_mutex_lock(&c->mutex); if (c->root) iter(c->root, min, max, func, privdata); util_mutex_unlock(&c->mutex); } critnib-1.1/critnib.h000066400000000000000000000015671415072002200146120ustar00rootroot00000000000000/* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright 2018, Intel Corporation */ #ifndef CRITNIB_H #define CRITNIB_H 1 #include #ifdef __cplusplus extern "C" { #endif struct critnib; typedef struct critnib critnib; enum find_dir_t { FIND_L = -2, FIND_LE = -1, FIND_EQ = 0, FIND_GE = +1, FIND_G = +2, }; critnib *critnib_new(void); void critnib_delete(critnib *c); int critnib_insert(critnib *c, uintptr_t key, void *value, int update); void *critnib_remove(critnib *c, uintptr_t key); void *critnib_get(critnib *c, uintptr_t key); void *critnib_find_le(critnib *c, uintptr_t key); int critnib_find(critnib *c, uintptr_t key, enum find_dir_t dir, uintptr_t *rkey, void **rvalue); void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, int (*func)(uintptr_t key, void *value, void *privdata), void *privdata); #ifdef __cplusplus } #endif #endif critnib-1.1/libcritnib.pod000066400000000000000000000046141415072002200156300ustar00rootroot00000000000000=encoding utf8 =head1 NAME libcritnib - an ordered map data structure with lock-free reads =head1 SYNOPSIS B<#include Ecritnib.hE> Link with I<-lcritnib>. =head1 DESCRIPTION =head2 Functions: =over =item B Creates a new empty critnib structure. =item BIB<);> Destroys and frees the memory. Note that removed items are reused but won't have their memory freed until this function is called. =item BIB<, uintptr_t >IB<, void *>IB<, int >IB<);> Adds a key:value pair to the critnib structure. If I is non-zero, an already existing key has its value updated, otherwise the function returns B. It may return B if we're out of memory, or 0 if all went okay. =item BIB<, uintptr_t >IB<);> Removes a given key from the structure. Its associated value is returned, or 0 (NULL) if there was no such key. =item BIB<, uintptr_t >IB<);> Obtains a value for a given key, or 0 (NULL) if not present. =item BIB<, uintptr_t >IB<);> Searches for the largest key not exceeding the argument, and returns its value. =item BIB<, uintptr_t >IB<, enum find_dir_t >IB<, uintptr_t *>IB<, void **>IB<);> Searches for a key that's smaller (B), smaller-or-equal (B), equal (B), greater-or-equal (B), or greater (B) than the argument. If found, the key and value are assigned to I<*rkey> and I<*rvalue> (which may be null to skip assignment), and 1 is returned. =item BIB<, uintptr_t >IB<, uintptr_t >IB<, >IB<, void *>IB<);> Walks the structure, visiting all entries whose keys are at least I but no larger than I (give B<-1> for no max), calling I for every entry found. If the I returns a non-zero value, the walk is aborted. The prototype for I should be: BIB<)(uintptr_t >IB<, void *>IB<, void *>IB<);> where I is an optional value passed to the iterator. B =back critnib-1.1/main.c000066400000000000000000000012251415072002200140660ustar00rootroot00000000000000#include #include #include "critnib.h" int main() { critnib *c = critnib_new(); critnib_insert(c, 1, "one", 0); critnib_insert(c, 2, "bad", 0); printf("overwrite(, 0) → %s\n", strerror(critnib_insert(c, 2, "foo", 0))); printf("overwrite(, 1) → %s\n", strerror(critnib_insert(c, 2, "two", 1))); printf("1 → %s\n", (char*)critnib_get(c, 1)); printf("2 → %s\n", (char*)critnib_get(c, 2)); printf("≤1 → %s\n", (char*)critnib_find_le(c, 1)); printf("≤2 → %s\n", (char*)critnib_find_le(c, 2)); printf("≤3 → %s\n", (char*)critnib_find_le(c, 3)); critnib_delete(c); return 0; } critnib-1.1/pmdk-compat.h000066400000000000000000000020441415072002200153630ustar00rootroot00000000000000#include #include #include #include #include typedef pthread_mutex_t os_mutex_t; #define Malloc malloc #define Free free static void *Zalloc(size_t s) { void *m = Malloc(s); if (m) memset(m, 0, s); return m; } #define ERR(x) do fprintf(stderr, x); while(0) #define util_mutex_init(x) pthread_mutex_init(x, NULL) #define util_mutex_destroy(x) pthread_mutex_destroy(x) #define util_mutex_lock(x) pthread_mutex_lock(x) #define util_mutex_unlock(x) pthread_mutex_unlock(x) #define util_lssb_index64(x) ((unsigned char)__builtin_ctzll(x)) #define util_mssb_index64(x) ((unsigned char)(63 - __builtin_clzll(x))) #define util_lssb_index32(x) ((unsigned char)__builtin_ctzl(x)) #define util_mssb_index32(x) ((unsigned char)(31 - __builtin_clzl(x))) #define NOFUNCTION do ; while(0) // Make these an unthing for now... #define ASSERT(x) NOFUNCTION #define ASSERTne(x, y) ASSERT(x != y) #define VALGRIND_ANNOTATE_NEW_MEMORY(p, s) NOFUNCTION #define VALGRIND_HG_DRD_DISABLE_CHECKING(p, s) NOFUNCTION critnib-1.1/tests/000077500000000000000000000000001415072002200141405ustar00rootroot00000000000000critnib-1.1/tests/.gitignore000066400000000000000000000000431415072002200161250ustar00rootroot000000000000001corr inequal th iter-cli iter.out critnib-1.1/tests/1corr.c000066400000000000000000000102751415072002200153370ustar00rootroot00000000000000#include #include #include "rand.h" #include "hmproto.h" #define ARRAYSZ(x) (sizeof(x)/sizeof(x[0])) typedef uintptr_t word; static int bad=0; #define CHECK(x,y) do if ((x)!=(y)) \ printf("\e[31mWRONG: \e[1m%s\e[22m (\e[1m%zx\e[22m) ≠ \e[1m%s\e[22m (\e[1m%zx\e[22m) at line \e[1m%d\e[22m\n", \ #x, (uintptr_t)(x), #y, (uintptr_t)(y), __LINE__),bad=1,exit(1); while (0) static void test_smoke() { void *c = hm_new(); hm_insert(c, 123, (void*)456, 0); CHECK(hm_get(c, 123), (void*)456); CHECK(hm_get(c, 124), 0); hm_delete(c); } static void test_key0() { void *c = hm_new(); hm_insert(c, 1, (void*)1, 0); hm_insert(c, 0, (void*)2, 0); hm_insert(c, 65536, (void*)3, 0); CHECK(hm_get(c, 1) , (void*)1); CHECK(hm_remove(c, 1), (void*)1); CHECK(hm_get(c, 0) , (void*)2); CHECK(hm_remove(c, 0) , (void*)2); CHECK(hm_get(c, 65536) , (void*)3); CHECK(hm_remove(c, 65536), (void*)3); hm_delete(c); } static void test_1to1000() { void *c = hm_new(); for (long i=0; i<1000; i++) hm_insert(c, i, (void*)i, 0); for (long i=0; i<1000; i++) CHECK(hm_get(c, i), (void*)i); hm_delete(c); } static void test_insert_delete1M() { #define MAX 1048576 void *c = hm_new(); for (long i=0; i #include #include "hmproto.h" struct hm hms[] = { // HM_ARR(critbit, 2), // HM_ARR(tcradix, 2), HM_ARR(critnib, 0), // HM_ARR(critnib_tag, 0), }; void hm_select(int i) { hm_new = hms[i].hm_new; hm_delete = hms[i].hm_delete; hm_insert = hms[i].hm_insert; hm_remove = hms[i].hm_remove; hm_get = hms[i].hm_get; hm_find_le = hms[i].hm_find_le; hm_find = hms[i].hm_find; hm_name = hms[i].hm_name; hm_immutable= hms[i].hm_immutable; } critnib-1.1/tests/hmproto.h000066400000000000000000000034351415072002200160060ustar00rootroot00000000000000#include enum find_dir_t { FIND_L = -2, FIND_LE = -1, FIND_EQ = 0, FIND_GE = +1, FIND_G = +2, }; #define HM_PROTOS(x) \ void *x##_new(void);\ void x##_delete(void *c);\ \ int x##_insert(void *c, uintptr_t key, void *value, int update);\ void *x##_remove(void *c, uintptr_t key);\ void *x##_get(void *c, uintptr_t key);\ void *x##_find_le(void *c, uintptr_t key);\ int x##_find(void *c, uintptr_t key, int dir, uintptr_t *rkey, void *rvalue); //HM_PROTOS(critbit) //HM_PROTOS(tcradix) HM_PROTOS(critnib) //HM_PROTOS(critnib_tag) void *(*hm_new)(void); void (*hm_delete)(void *c); int (*hm_insert)(void *c, uintptr_t key, void *value, int update); void *(*hm_remove)(void *c, uintptr_t key); void *(*hm_get)(void *c, uintptr_t key); void *(*hm_find_le)(void *c, uintptr_t key); int (*hm_find)(void *c, uintptr_t key, int dir, uintptr_t *rkey, void *rvalue); const char *hm_name; int hm_immutable; #define HM_SELECT_ONE(x,f) hm_##f=x##_##f #define HM_SELECT(x) \ HM_SELECT_ONE(x,new);\ HM_SELECT_ONE(x,delete);\ HM_SELECT_ONE(x,insert);\ HM_SELECT_ONE(x,remove);\ HM_SELECT_ONE(x,get);\ HM_SELECT_ONE(x,find_le);\ HM_SELECT_ONE(x,find);\ hm_name=#x #define HM_ARR(x,imm) { x##_new, x##_delete, x##_insert, x##_remove, x##_get, \ x##_find_le, x##_find, #x, imm } struct hm { void *(*hm_new)(void); void (*hm_delete)(void *c); int (*hm_insert)(void *c, uintptr_t key, void *value, int update); void *(*hm_remove)(void *c, uintptr_t key); void *(*hm_get)(void *c, uintptr_t key); void *(*hm_find_le)(void *c, uintptr_t key); int (*hm_find)(void *c, uintptr_t key, int dir, uintptr_t *rkey, void *rvalue); const char *hm_name; int hm_immutable; } hms[1]; void hm_select(int i); critnib-1.1/tests/inequal.c000066400000000000000000000070441415072002200157470ustar00rootroot00000000000000#include #include #include "rand.h" #include "hmproto.h" #define ARRAYSZ(x) (sizeof(x)/sizeof(x[0])) typedef uintptr_t word; static int bad=0; #define CHECK(x,y) do if ((x)!=(y)) \ printf("\e[31mWRONG: \e[1m%s\e[22m (\e[1m%zx\e[22m) ≠ \e[1m%s\e[22m (\e[1m%zx\e[22m) at line \e[1m%d\e[22m\n", \ #x, (uintptr_t)(x), #y, (uintptr_t)(y), __LINE__),bad=1,exit(1); while (0) static void test_le_basic() { void *c = hm_new(); #define INS(x) hm_insert(c, (x), (void*)(x), 0) INS(1); INS(2); INS(3); INS(0); INS(4); INS(0xf); INS(0xe); INS(0x11); INS(0x12); INS(0x20); #define GET_SAME(x) CHECK(hm_get(c, (x)), (void*)(x)) #define GET_NULL(x) CHECK(hm_get(c, (x)), NULL) GET_NULL(122); GET_SAME(1); GET_SAME(2); GET_SAME(3); GET_SAME(4); GET_NULL(5); GET_SAME(0x11); GET_SAME(0x12); #define LE(x,y) CHECK(hm_find_le(c, (x)), (void*)(y)) LE(1, 1); LE(2, 2); LE(5, 4); LE(6, 4); LE(0x11, 0x11); LE(0x15, 0x12); LE(0xfffffff, 0x20); hm_delete(c); } static word expand_bits(word x) { return (x&0xc000)<<14 | (x&0x3000)<<12 | (x&0x0c00)<<10 | (x&0x0300)<< 8 | (x&0x00c0)<< 6 | (x&0x0030)<< 4 | (x&0x000c)<< 2 | (x&0x0003); } static void test_le_brute() { void *c = hm_new(); char ws[65536]={0,}; for (int cnt=0; cnt<1024; cnt++) { int w = rnd64()&0xffff; if (ws[w]) hm_remove(c, expand_bits(w)), ws[w]=0; else hm_insert(c, expand_bits(w), (void*)expand_bits(w), 0), ws[w]=1; for (int cnt2=0; cnt2<1024; cnt2++) { w = rnd64()&0xffff; int v; for (v=w; v>=0 && !ws[v]; v--) ; word res = (word)hm_find_le(c, expand_bits(w)); word exp = (v>=0)?expand_bits(v):0; CHECK(res, exp); void *W; if (hm_find(c, expand_bits(w), FIND_LE, 0, &W)) res = (intptr_t)W; else res = 0; CHECK(res, exp); } } hm_delete(c); } static void test_ge_brute() { void *c = hm_new(); char ws[65536]={0,}; for (int cnt=0; cnt<1024; cnt++) { int w = rnd64()&0xffff; if (ws[w]) hm_remove(c, expand_bits(w)), ws[w]=0; else hm_insert(c, expand_bits(w), (void*)expand_bits(w), 0), ws[w]=1; for (int cnt2=0; cnt2<1024; cnt2++) { w = rnd64()&0xffff; int v; for (v=w; v<0x10000 && !ws[v]; v++) ; word res; word exp = (v<0x10000)?expand_bits(v):0; void *W; if (hm_find(c, expand_bits(w), FIND_GE, 0, &W)) res = (intptr_t)W; else res = 0; CHECK(res, exp); } } hm_delete(c); } static void run_test(void (*func)(void), const char *name, int req) { printf("TEST: %s\n", name); for (int i=0; iiter.out <iter.out <iter.out <iter.out <iter.out <iter.out <iter.out < #include #include #include "../critnib.h" #define die(...) do {fprintf(stderr, __VA_ARGS__); exit(1);} while(0) static int itf(uintptr_t key, void *value, void *dummy) { printf("%lu\n", key); return (key == 42); } int main(int argc, char **argv) { if (argc != 3) die("Need two args: min, max\n"); uintptr_t min = atol(argv[1]); uintptr_t max = atol(argv[2]); critnib *c = critnib_new(); uintptr_t x; int err; while (scanf("%lu", &x) == 1) if ((err = critnib_insert(c, x, (void*)x, 0))) { if (err == EEXIST) printf("dupe: %lu\n", x); else die("critnib_insert failed\n"); } critnib_iter(c, min, max, itf, 0); critnib_delete(c); return 0; } critnib-1.1/tests/rand.c000066400000000000000000000046251415072002200152370ustar00rootroot00000000000000// SPDX-License-Identifier: BSD-3-Clause /* Copyright 2019, Intel Corporation */ /* * rand.c -- random utils */ #include #include #include #include #include #include "rand.h" #ifdef _WIN32 # include # include #else # include # include # ifdef __APPLE__ # include # endif #endif /* * hash64 -- a u64 -> u64 hash */ uint64_t hash64(uint64_t x) { x += 0x9e3779b97f4a7c15; x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; x = (x ^ (x >> 27)) * 0x94d049bb133111eb; return x ^ (x >> 31); } /* * xoshiro256** random generator * * Fastest available good PRNG as of 2018 (sub-nanosecond per entry), produces * much better output than old stuff like rand() or Mersenne's Twister. * * By David Blackman and Sebastiano Vigna; PD/CC0 2018. * * It has a period of 2²⁵⁶-1, excluding all-zero state; it must always get * initialized to avoid that zero. */ static inline uint64_t rotl(const uint64_t x, int k) { /* optimized to a single instruction on x86 */ return (x << k) | (x >> (64 - k)); } /* * rnd64_r -- return 64-bits of randomness */ uint64_t rnd64_r(rng_t *state) { uint64_t *s = (void *)state; const uint64_t result = rotl(s[1] * 5, 7) * 9; const uint64_t t = s[1] << 17; s[2] ^= s[0]; s[3] ^= s[1]; s[1] ^= s[2]; s[0] ^= s[3]; s[2] ^= t; s[3] = rotl(s[3], 45); return result; } /* * randomize_r -- initialize random generator * * Seed of 0 means random. */ void randomize_r(rng_t *state, uint64_t seed) { if (!seed) { #ifdef SYS_getrandom /* We want getentropy() but ancient Red Hat lacks it. */ if (syscall(SYS_getrandom, state, sizeof(rng_t), 0) == sizeof(rng_t)) { return; /* nofail, but ENOSYS on kernel < 3.16 */ } #elif _WIN32 #pragma comment(lib, "Bcrypt.lib") if (BCryptGenRandom(NULL, (PUCHAR)state, sizeof(rng_t), BCRYPT_USE_SYSTEM_PREFERRED_RNG)) { return; } #else if (!getentropy(state, sizeof(rng_t))) return; #endif // best effort fallback seed = (uint64_t)pthread_self(); } uint64_t *s = (void *)state; s[0] = hash64(seed); s[1] = hash64(s[0]); s[2] = hash64(s[1]); s[3] = hash64(s[2]); } static rng_t global_rng; /* * rnd64 -- global state version of rnd64_t */ uint64_t rnd64(void) { return rnd64_r(&global_rng); } /* * randomize -- initialize global RNG */ void randomize(uint64_t seed) { randomize_r(&global_rng, seed); } critnib-1.1/tests/rand.h000066400000000000000000000006701415072002200152400ustar00rootroot00000000000000/* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright 2019-2020, Intel Corporation */ /* * rand.h -- random utils */ #ifndef RAND_H #define RAND_H 1 #include #ifdef __cplusplus extern "C" { #endif typedef uint64_t rng_t[4]; uint64_t hash64(uint64_t x); uint64_t rnd64_r(rng_t *rng); void randomize_r(rng_t *rng, uint64_t seed); uint64_t rnd64(void); void randomize(uint64_t seed); #ifdef __cplusplus } #endif #endif critnib-1.1/tests/th.c000066400000000000000000000215621415072002200147250ustar00rootroot00000000000000#define _GNU_SOURCE #include #include #include #include #include "rand.h" #include "hmproto.h" #define ARRAYSZ(x) (sizeof(x)/sizeof(x[0])) typedef uintptr_t word; // These functions return a 64-bit value on 64-bit systems; I'd be // interested in testing stuff on a box where you have more cores // than fits in an uint32_t... static word nthreads, nrthreads, nwthreads; static word nproc() { #if defined(__USE_GNU) && !defined(__gnu_hurd__) cpu_set_t set; if (!pthread_getaffinity_np(pthread_self(), sizeof(set), &set)) return CPU_COUNT(&set); #endif long n = sysconf(_SC_NPROCESSORS_CONF); if (n > 0) return n; return 0; } static int bad=0, any_bad=0; #define CHECK(x) do if (!(x)) bad=1; while (0) #define CHECKP(x,...) do if (!(x)) {bad=1; printf("ERROR: "__VA_ARGS__);} while (0) static int done=0; static word the1000[1000]; static void* the1000p[1000]; #if __SIZEOF_SIZE_T__ == 8 # define K 0xdeadbeefcafebabe #else # define K 0xcafebabe #endif /***********/ /* threads */ /***********/ static void* thread_read1(void* c) { word count=0; while (!done) { CHECK(hm_get(c, K) == (void*)K); count++; } return (void*)count; } static void* thread_read1p(void* c) { void* k = the1000p[0]; word count=0; while (!done) { CHECK(hm_get(c, (word)k) == k); count++; } return (void*)count; } static void* thread_read1000(void* c) { word count=0; int i=0; while (!done) { if (++i==1000) i=0; word v=the1000[i]; CHECK(hm_get(c, v) == (void*)v); count++; } return (void*)count; } static void* thread_write1000(void* c) { rng_t rng; randomize_r(&rng, 0); word w1000[1000]; for (int i=0; i>=1) { if (x & a) y |= b; } return y; } static void* thread_le1(void* c) { word count=0; while (!done) { word y = revbits(count); if (y < K) CHECK(hm_find_le(c, y) == NULL); else CHECK(hm_find_le(c, y) == (void*)K); count++; } return (void*)count; } static void* thread_le1000(void* c) { word count=0; while (!done) { word y = revbits(count); hm_find_le(c, y); count++; } return (void*)count; } /*********/ /* tests */ /*********/ typedef void *(*thread_func_t)(void *); static void run_test(int spreload, int rpreload, thread_func_t rthread, thread_func_t wthread) { int ptrs = (rpreload<0); void *c = hm_new(); if (spreload>=1) hm_insert(c, K, (void*)K, 0); if (spreload>=2) hm_insert(c, 1, (void*)1, 0); if (ptrs) { rpreload=-rpreload; for (int i=spreload; i=0 && only_hm