pax_global_header00006660000000000000000000000064141402105230014502gustar00rootroot0000000000000052 comment=9ae3109cfe17b11be7fc5c8f72cd1e333a48d11c persistent-cache-cpp-1.0.5/000077500000000000000000000000001414021052300155265ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/.gitignore000066400000000000000000000001621414021052300175150ustar00rootroot00000000000000# default build dir /build* /obj-* # Eclipse project files /.project /.cproject /.settings /.pydevproject *.user persistent-cache-cpp-1.0.5/AUTHORS000066400000000000000000000001341414021052300165740ustar00rootroot00000000000000Dalton Durst Marius Gripsgard Michi Henning Mike Gabriel Pete Woods Ratchanan Srirattanamet persistent-cache-cpp-1.0.5/CMakeLists.txt000066400000000000000000000104351414021052300202710ustar00rootroot00000000000000cmake_minimum_required(VERSION 2.8.11) # Default install location. Must be set here, before setting the project. if (NOT DEFINED CMAKE_INSTALL_PREFIX) set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install CACHE PATH "" FORCE) endif() project(persistent-cache-cpp C CXX) set(CMAKE_INCLUDE_CURRENT_DIR ON) set(LIBNAME persistent-cache-cpp) set(VERSION_MAJOR 1) set(VERSION_MINOR 0) set(VERSION_MICRO 5) set(LIBVERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_MICRO}") string(TOLOWER "${CMAKE_BUILD_TYPE}" cmake_build_type_lower) # Build types should always be lower case set(ACCEPTED_BUILD_TYPES "" none release debug relwithdebinfo coverage) list(FIND ACCEPTED_BUILD_TYPES "${cmake_build_type_lower}" IS_BUILD_TYPE_ACCEPTED) if(${IS_BUILD_TYPE_ACCEPTED} EQUAL -1) message(FATAL_ERROR "Invalid CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}\nValid types are: ${ACCEPTED_BUILD_TYPES}") endif() set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -Wall -pedantic -Wextra") # Some additional warnings not included by the general flags set above. set(EXTRA_C_WARNINGS "-Wcast-align -Wcast-qual -Wformat -Wredundant-decls -Wswitch-default") set(EXTRA_CXX_WARNINGS "-Wnon-virtual-dtor -Wctor-dtor-privacy -Wold-style-cast") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_C_WARNINGS} ${EXTRA_CXX_WARNINGS}") # By default, for release builds, warnings become hard errors. if ("${cmake_build_type_lower}" STREQUAL "release" OR "${cmake_build_type_lower}" STREQUAL "relwithdebinfo") option(Werror "Treat warnings as errors" ON) else() option(Werror "Treat warnings as errors" OFF) endif() # If warnings are errors, don't error on deprecated declarations. if (${Werror}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") if ("${cmake_build_type_lower}" STREQUAL "release" OR "${cmake_build_type_lower}" STREQUAL "relwithdebinfo") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations") endif() endif() # Flags for thread/address sanitizer set(SANITIZER "" CACHE STRING "Build with -fsanitize= (legal values: thread, address)") if ("${SANITIZER}" STREQUAL "") # Do nothing elseif (${SANITIZER} STREQUAL "thread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fno-omit-frame-pointer -g") elseif (${SANITIZER} STREQUAL "address") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -g") else() message(FATAL_ERROR "Invalid SANITIZER setting: ${SANITIZER}") endif() # Some tests are slow, so make it possible not to run them # during day-to-day development. option(slowtests "Run slow tests" ON) # Definitions for testing with valgrind. configure_file(CTestCustom.cmake.in CTestCustom.cmake) # Tests in CTestCustom.cmake are skipped for valgrind find_program(MEMORYCHECK_COMMAND NAMES valgrind) if (MEMORYCHECK_COMMAND) set(MEMORYCHECK_COMMAND_OPTIONS "--suppressions=${CMAKE_SOURCE_DIR}/valgrind-suppress --errors-for-leak-kinds=definite --show-leak-kinds=definite --leak-check=full --num-callers=50 --error-exitcode=3" ) add_custom_target(valgrind DEPENDS NightlyMemCheck) else() message(WARNING "Cannot find valgrind: valgrind target will not be available") endif() include(CTest) enable_testing() find_package(CoverageReport) find_package(Boost COMPONENTS filesystem REQUIRED) find_package(Threads REQUIRED) include_directories(include) add_subdirectory(src) add_subdirectory(include) add_subdirectory(tests) add_subdirectory(data) add_subdirectory(examples) add_subdirectory(doc) enable_coverage_report( TARGETS ${LIBNAME} ${UNIT_TEST_TARGETS} FILTER ${CMAKE_SOURCE_DIR}/tests/* ${CMAKE_BINARY_DIR}/* TESTS ${UNIT_TEST_TARGETS} ) find_package(DoxygenBuilder) file(GLOB public_headers ${CMAKE_SOURCE_DIR}/include/core/*.h) add_doxygen( ${LIBNAME}-doc INPUT ${CMAKE_BINARY_DIR}/doc/main_page.dox ${public_headers} OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/doc STRIP_FROM_PATH "${CMAKE_SOURCE_DIR}/src" STRIP_FROM_INC_PATH "${CMAKE_SOURCE_DIR}/include" EXCLUDE_PATTERNS */internal/* EXCLUDE_SYMBOLS *::internal* *::Priv ALL ) install(DIRECTORY ${CMAKE_BINARY_DIR}/doc/html DESTINATION share/doc/${LIBNAME}) persistent-cache-cpp-1.0.5/COPYING.LGPL000066400000000000000000000167271414021052300173330ustar00rootroot00000000000000 GNU LESSER GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 0. Additional Definitions. As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 1. Exception to Section 3 of the GNU GPL. You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 2. Conveying Modified Versions. If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 3. Object Code Incorporating Material from Library Header Files. The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the object code with a copy of the GNU GPL and this license document. 4. Combined Works. You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the Combined Work with a copy of the GNU GPL and this license document. c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. d) Do one of the following: 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 5. Combined Libraries. You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 6. Revised Versions of the GNU Lesser General Public License. The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. persistent-cache-cpp-1.0.5/CTestCustom.cmake.in000066400000000000000000000006741414021052300213610ustar00rootroot00000000000000# # Tests listed here will not be run by the valgrind target, # either because there is not point (we don't want to # test that a python script doesn't leak), or because, # under valgrind, the test runs too slowly to meet # its timing constraints (or crashes valgrind). # SET(CTEST_CUSTOM_MEMCHECK_IGNORE copyright speed stand-alone-core-headers stand-alone-core-internal-headers clean-public-core-headers whitespace ) persistent-cache-cpp-1.0.5/ChangeLog000066400000000000000000000236331414021052300173070ustar00rootroot000000000000002021-11-02 Ratchanan Srirattanamet * Release 1.0.5 (HEAD -> releases/1.0.5, tag: 1.0.5) * Merge branch 'personal/peat-psuwit/skip-non-root-test' into 'master' (fa809bf) 2021-10-06 Dalton Durst * Merge branch 'personal/peat-psuwit/gitignore-obj' into 'master' (7938174) 2021-10-06 Ratchanan Srirattanamet * tests: skip permission denied test if running as root (4e2c06c) Fixes: https://gitlab.com/ubports/core/lib-cpp/persistent-cache-cpp/-/issues/1 2021-09-21 Ratchanan Srirattanamet * Merge branch 'mr/fix-copyright-tests-on-some-occasions' into 'master' (225e19d) * .gitignore: ignore obj-*, Debian build directory (36fc366) 2021-09-18 Mike Gabriel * debian/control: Add to B-D: licensecheck. (8f59bf2) * tests/copyright/check_copyright.sh: Ignore Git related and quilt related directory/file structures. (e940fc8) 2021-07-19 Ratchanan Srirattanamet * Merge branch 'mr/debian-control-libgtest-dev' into 'master' (c9d6dc2) 2021-07-17 Mike Gabriel * debian/control: Add B-D libgtest-dev. Fix FTBFS (No rule to make target 'tests/core/gmock/libgmock_main.a', needed by ...) on Debian 11 and above. (66b0628) 2020-07-12 Marius Gripsgard * Fix gmock (57ccf1a) * Import and clean (7bfd592) 2017-01-13 Michi Henning * Merged trunk. (d253ac8) * Merged lp:~michihenning/persistent-cache-cpp/merge-crossbuild (0f719a7) * Merged lp:~pete-woods/persistent-cache-cpp/cmake-extras-compatibility (b35dd0f) 2016-12-07 Pete Woods * Compatibility with unversioned cmake-extras modules (812c87c) 2016-11-29 Michi Henning * Updated CMakeLists.txt to work with gtest 1.8. Suppressed warnings from gtests. (fcfb860) 2016-11-28 Michi Henning * Replaced libgtest-dev with google-mock build test. (809464b) * Moved GMock find_package one level down. (f1bdbb4) * Fixed failing copyright test. (9fd3e00) * Updated cmake files to work with gtest 1.8. Suppressed warnings from gtest. (a98d164) 2016-07-25 Michi Henning * Changed python dependency to python for cross-builds. (93487f1) 2016-07-21 Michi Henning * Merged lp:~michihenning/persistent-cache-cpp/fix-cross-build: Changed python dependency to python:any for cross-builds. (c2cf024) 2016-07-20 Michi Henning * Changed python dependency to python:any for cross-builds. (b528e8d) 2016-05-09 Michi Henning * No-change commit to test Jenkins. (15817f3) * No-change commit for Jenkins testing. (639b63c) 2016-05-06 Michi Henning * Back-merge of 1.0.4+16.10.20160505.5-0ubuntu1 from trunk. (b42bb76) * Back-merge of 1.0.4+16.10.20160505.5-0ubuntu1 from trunk. (d50fa68) 2016-05-05 CI Train Bot * Releasing 1.0.4+16.10.20160505.5-0ubuntu1 (3863ce9) 2016-05-05 Michi Henning * Fix for copyright test. (9b808c8) * No-change rebuild to pick up -fPIE on amd64. (cf955a8) * Merged fixes from devel. (0f9bb58) * Fixed copyright test by adding new suppressions. (396535c) * Added missing suppressions to copyright test. (fc456f3) 2016-01-19 Michi Henning * Merge 1.0.4+16.04.20160117-0ubuntu1 from trunk after release. (9e5100e) 2016-01-20 Michi Henning * Merge trunk after release. (c266acf) 2016-01-17 CI Train Bot * Releasing 1.0.4+16.04.20160117-0ubuntu1 (74dce9d) 2016-01-17 Michi Henning * Fix for assertion failure in stats move constructor. Approved by: Michi Henning (ad77992) 2016-01-18 Michi Henning * Merged devel for silo landing. (955b434) 2016-01-16 Michi Henning * Fixed assertion failure when move-constructing from internal stats instance. (de65360) * Work-around for broken licensecheck on xenial. (8a9659d) 2015-12-10 Michi Henning * Merged dependent branch. (5385775) * Add work-around for broken licensecheck on xenial. (351880a) 2015-12-09 Michi Henning * Fixed typo in comment. (41edd95) * Updated changelog. (4174faf) * Fixed assertion failure when move-constructing from internal stats instance. (d3cdf73) 2015-08-28 Michi Henning * Merged 1.0.4+15.10.20150819.2-0ubuntu1 from trunk. (826653c) * Merged 1.0.4+15.10.20150819.2-0ubuntu1 from trunk. (44e07f4) 2015-08-19 Michi Henning * Fixed wrong performance claim due to insufficient number of iterations. (51513ff) 2015-08-19 CI Train Bot * Releasing 1.0.4+15.10.20150819.2-0ubuntu1 (5d4d4a4) 2015-08-19 Michi Henning * Minor enhancements to cache. Suppressed a lintian warning, fixed autopkg test dependency, and added run length averages to stats. Approved by: PS Jenkins bot (9e40a20) 2015-08-20 Michi Henning * Merged documentation fix. (1bd4764) * Fixed changelog again. (c4b4c8d) 2015-08-19 Michi Henning * Fixed changelog version and change micro version to match. (08cc661) * Landing branch devel -> trunk. (61afe12) * Added average hit and miss run stats. (f6a886a) * Merged devel and resolved conflict. Yet another try at getting rid of the garbage in changelog. (fe5db4c) * Simplified avg calculation (thanks James!) (50ed182) * Added examples to the unit tests, so we can be sure they don't break. (8d0d8b6) * Merging trunk into devel after releasing 1.0.3. (80e7c0f) * Added average hit and miss run stats. (470f47a) 2015-08-17 Michi Henning * Fixed incorrect performance claim due to insufficient number of iterations. (d274de1) 2015-08-10 Michi Henning * Fixed whitespace in CMakeLists.txt (59ddde4) 2015-08-08 Michi Henning * Added examples to the unit tests, so we can be sure they don't break. (ef867fa) * Merging trunk into devel after releasing 1.0.3. (cc69f34) 2015-08-07 CI Train Bot * Releasing 1.0.3+15.10.20150807-0ubuntu1 (0aa341d) 2015-08-07 Michi Henning * Merging fixes for autopkg test from devel. Approved by: Michi Henning (3d7e1e4) * Merging devel into trunk for landing. (d459715) * Some more fixes for autopkg test. Passes under qemu now. (d516339) * Now passed under adt-run. (fddbfdb) * Disabled env vars in autopkg test. (1d6ce84) * Fixed another dependency. More trace. (bfaadb2) * Test needs to depend on libleveldb-dev. (8be4edd) * Testing dependencies again. (c7ca6a8) * More hackery for autopkg test. (18001d6) * Trying pkg-config with absolute path. (2f32a2e) * Added missing dependency to autopkg test. (c7d2e49) * Added missing dependency to autopkg test. (e49a084) * Merged devel. (4be954c) * Suppressed lintian warning about doxygen's use of jquery.js. (cd07eee) 2015-08-06 Michi Henning * Suppressed lintian warning about doxygen's use of jquery.js. (3796d7b) * Merged trunk to get debian/changelog up to date. Remove noise in changelog caused by bug in the packaging machinery. (8644ad7) * Added lintian suppression for doxygen's use of jquery.js. (a5ced02) * Merged trunk and cleaned up messy changelog caused by bug in the packaging machinery. (c5c6f5e) 2015-08-05 CI Train Bot * Releasing 1.0.0+15.10.20150805-0ubuntu1 (c4c1d2b) 2015-08-05 Michi Henning * Initial revision for landing. Approved by: PS Jenkins bot, James Henstridge, Michi Henning (41522bd) * Fixes for gcc 5 and new changelog entry for packaging. (7879748) 2015-07-29 Michi Henning * Updated autopkg test for fixed pkg-config file. (277ca1a) * Fixed missing lib in pkg-config. (57e6263) * Added python3 to build deps. (ccc70e5) * Added config for split building. (a5e58bd) * Updated version number in changelog. (d3dded3) * Fixed TODO. (f228eeb) * Merged devel: Michi Henning 2015-07-29 Trying without -DLIBDIR Michi Henning 2015-07-29 Added more missing deps. Michi Henning 2015-07-29 Don't need click to build. Michi Henning 2015-07-29 Added pkgconfig to deps. Michi Henning 2015-07-29 [merge] pending merges: Michi Henning 2015-07-29 Added cmake to dependencies. (9a72508) * Trying without -DLIBDIR (4f2d26f) * Added more missing deps. (f0242f3) * Don't need click to build. (1fbc2f4) * Added pkgconfig to deps. (f5e6bc3) * pending merges: Michi Henning 2015-07-28 [merge] Doc fix. Michi Henning 2015-07-28 [merge] Merged devel: (f5542db) * Added cmake to dependencies. (54945db) * Added cmake to dependencies. (f2ea7cc) 2015-07-28 Michi Henning * Doc fix. (2b438ff) * Doc fix. (2d3ee93) * Merged devel: More doc improvements. Fleshed out string cache example. Doc improvements. (9d0e63e) * More doc improvements. (13a8aea) * Fleshed out string cache example. Doc improvements. (9c1a071) 2015-07-27 Michi Henning * Minor fix to test script. (18ca89c) * Enabled autopkgtest. (6f44258) * Added autopkgtest. (1c62fff) * Added examples directory. (0787ad5) 2015-07-24 Michi Henning * Added cmake-extras to build deps. (d4aca03) * Added pkgconfig support. (0388de2) 2015-07-23 Michi Henning * Added doc package. (48aaa38) * Fixed incorrect include file install dir. (234ca00) * Bunch of debian adjustments. (37db4fb) * Started on debian packaging. (27ac6e7) * Removed build directory (checked in accidentally). (bf556ff) 2015-07-21 Michi Henning * Initial version from thumbnailer, with one minor bug fix. (4d1b27d) persistent-cache-cpp-1.0.5/HACKING000066400000000000000000000075551414021052300165310ustar00rootroot00000000000000Building the code ----------------- By default, the code is built in release mode. To build a debug version, use $ mkdir builddebug $ cd builddebug $ cmake -DCMAKE_BUILD_TYPE=debug .. $ make For a release version, use -DCMAKE_BUILD_TYPE=release Running the tests ----------------- $ make $ make test Note that "make test" alone is dangerous because it does not rebuild any tests if either the library or the test files themselves need rebuilding. It's not possible to fix this with cmake because cmake cannot add build dependencies to built-in targets. To make sure that everything is up-to-date, run "make" before running "make test"! To run the tests with valgrind: $ make valgrind It doesn't make sense for some tests to run them with valgrind. For example, the header compilation tests don't need valgrind because we'd just be testing that Python doesn't leak. There are also some tests that run too slow and time out under valgrind and, occasionally, valgrind crashes for a particular test. There are two ways to suppress tests: You can add a test name to CTestCustom.cmake.in to suppress that test completely. That makes sense for the header compilation tests, for example. If a specific test case in a test program causes a valgrind problem, you can selectively disable a section of code like this: #include if (!RUNNING_ON_VALGRIND) { // Code here crashes valgrind... } That way, the test will still be run as part of the normal "make test" target, but will be ommitted when running "make valgrind". Coverage -------- To build with the flags for coverage testing enabled and get coverage: $ mkdir buildcoverage $ cd buildcoverage $ cmake -DCMAKE_BUILD_TYPE=coverage $ make $ make test $ make coverage Unfortunately, it is not possible to get 100% coverage for some files, mainly due to gcc's generation of two destructors for dynamic and non- dynamic instances. For abstract base classes and for classes that prevent stack and static allocation, this causes one of the destructors to be reported as uncovered. There are also issues with some functions in header files that are incorrectly reported as uncovered due to inlining, as well as the impossibility of covering defensive assert(false) statements, such as an assert in the default branch of a switch, where the switch is meant to handle all possible cases explicitly. If you run a binary and get lots of warnings about a "merge mismatch for summaries", this is caused by having made changes to the source that add or remove code that was previously run, so the new coverage output cannot sensibly be merged into the old coverage output. You can get rid of this problem by running $ make clean-coverage This deletes all the .gcda files, allowing the merge to (sometimes) succeed again. If this doesn't work either, the only remedy is to do a clean build. Code style ---------- Please maintain the existing coding style. For details on the style, see lp:canonical-client-development-guidelines. We use a format tool that fixes a whole lot of issues regarding code style. See the HACKING file of lp:unity-scopes-api for details on the tool. Thread and address sanitizer ---------------------------- Set SANITIZER to "thread" or "address" to build with the corresponding sanitizer enabled. Some libraries cause a cause a large number of warnings from thread sanitizer. The tsan-suppress file supresses the benign race conditions we currently know about. To run the tests (from the build directory) with these suppressions enabled, use: $ TSAN_OPTIONS="suppressions=../tsan-suppress" make test If a test runs too slowly under address sanitizer, you can hide a section of code from address sanitzer with: #if defined(__has_feature) #if !__has_feature(address_sanitizer) // Code here takes forever under address sanitizer... #endif #endif persistent-cache-cpp-1.0.5/data/000077500000000000000000000000001414021052300164375ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/data/CMakeLists.txt000066400000000000000000000005231414021052300211770ustar00rootroot00000000000000# Set up package config. set(PKGCONFIG_NAME ${LIBNAME}) set(PKGCONFIG_DESCRIPTION "A cache of key-value pairs with persistent storage for C++") set(VERSION "1.0") configure_file(lib${LIBNAME}.pc.in lib${LIBNAME}.pc @ONLY) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib${LIBNAME}.pc DESTINATION lib/${CMAKE_LIBRARY_ARCHITECTURE}/pkgconfig) persistent-cache-cpp-1.0.5/data/libpersistent-cache-cpp.pc.in000066400000000000000000000004231414021052300240770ustar00rootroot00000000000000prefix=@CMAKE_INSTALL_PREFIX@ includedir=${prefix}/include libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ Name: lib@LIBNAME@ Description: Cache of key-value pairs with persistent storage for C++ Version: @LIBVERSION@ Libs: -L${libdir} -l@LIBNAME@ -lleveldb Cflags: -I${includedir} persistent-cache-cpp-1.0.5/debian/000077500000000000000000000000001414021052300167505ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/debian/changelog000066400000000000000000000033451414021052300206270ustar00rootroot00000000000000persistent-cache-cpp (1.0.5) unstable; urgency=medium * Upstream-provided Debian package for persistent-cache-cpp. See upstream ChangeLog for recent changes. -- UBports developers Tue, 02 Nov 2021 17:14:52 +0700 persistent-cache-cpp (1.0.4+16.10.20160505.5-0ubuntu1) yakkety; urgency=medium [ CI Train Bot ] * No-change rebuild. -- Michi Henning Thu, 05 May 2016 05:52:21 +0000 persistent-cache-cpp (1.0.4+16.04.20160117-0ubuntu2) yakkety; urgency=medium * No-change rebuild to pick up -fPIE on amd64. -- Graham Inggs Wed, 04 May 2016 17:55:45 +0200 persistent-cache-cpp (1.0.4+16.04.20160117-0ubuntu1) xenial; urgency=medium [ Michi Henning ] * Fixed assertion failure when move-constructing a stats instance from the internal instance. [ CI Train Bot ] * No-change rebuild. -- Michi Henning Sun, 17 Jan 2016 22:35:54 +0000 persistent-cache-cpp (1.0.4+15.10.20150819.2-0ubuntu1) wily; urgency=medium [ Michi Henning ] * Added average hit and miss run stats. [ CI Train Bot ] * New rebuild forced. -- CI Train Bot Wed, 19 Aug 2015 21:50:26 +0000 persistent-cache-cpp (1.0.3+15.10.20150807-0ubuntu1) wily; urgency=medium [ Michi Henning ] * Suppressed lintian warning about doxygen's use of jquery.js. * Added missing dependency to autopkg test. -- CI Train Bot Fri, 07 Aug 2015 11:09:03 +0000 persistent-cache-cpp (1.0.0+15.10.20150805-0ubuntu1) wily; urgency=medium [ Michi Henning ] * Initial release [ CI Train Bot ] * New rebuild forced. -- CI Train Bot Wed, 05 Aug 2015 00:21:58 +0000 persistent-cache-cpp-1.0.5/debian/compat000066400000000000000000000000021414021052300201460ustar00rootroot000000000000009 persistent-cache-cpp-1.0.5/debian/control000066400000000000000000000025141414021052300203550ustar00rootroot00000000000000Source: persistent-cache-cpp Priority: optional Maintainer: UBports developers Build-Depends: cmake, cmake-extras (>= 0.10), debhelper (>= 9), devscripts, doxygen, google-mock, libboost-filesystem-dev, libgtest-dev, libleveldb-dev, licensecheck, lsb-release, pkg-config, python3 , Standards-Version: 3.9.6 XS-Testsuite: autopkgtest Section: libs Homepage: https://gitlab.com/ubports/core/lib-cpp/persistent-cache-cpp Vcs-git: https://gitlab.com/ubports/core/lib-cpp/persistent-cache-cpp.git Vcs-Browser: https://gitlab.com/ubports/core/lib-cpp/persistent-cache-cpp Package: persistent-cache-cpp-dev Section: libdevel Architecture: any Multi-Arch: same Depends: ${misc:Depends}, libboost-dev, libleveldb-dev, pkg-config, Description: Cache of key-value pairs with persistent storage for C++ 11 A persistent cache for arbitrary (possibly large amount of data, such as image files) that is fast, scalable, and crash-proof. Package: persistent-cache-cpp-doc Section: doc Architecture: all Multi-Arch: foreign Depends: ${misc:Depends}, Description: Documentation for persistent-cache-cpp-dev Examples and API reference. persistent-cache-cpp-1.0.5/debian/copyright000066400000000000000000000017211414021052300207040ustar00rootroot00000000000000Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: persistent-cache-cpp Source: https://launchpad.net/persistent-cache-cpp Files: * Copyright: 2015 Canonical Ltd. License: LGPL-3 License: LGPL-3 This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, version 3 of the License. . This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. . You should have received a copy of the GNU Lesser General Public License along with this program. If not, see . . On Debian systems, the full text of the GNU General Public License version 3 can be found in the file /usr/share/common-licenses/LGPL-3. persistent-cache-cpp-1.0.5/debian/persistent-cache-cpp-dev.install000066400000000000000000000001221414021052300251300ustar00rootroot00000000000000usr/include/core/* /usr/lib/*/libpersistent-cache-cpp.a /usr/lib/*/pkgconfig/*.pc persistent-cache-cpp-1.0.5/debian/persistent-cache-cpp-doc.install000066400000000000000000000001251414021052300251220ustar00rootroot00000000000000/usr/share/doc/persistent-cache-cpp/* /usr/share/doc/persistent-cache-cpp/examples/* persistent-cache-cpp-1.0.5/debian/persistent-cache-cpp-doc.lintian-overrides000066400000000000000000000002511414021052300271120ustar00rootroot00000000000000# Nothing we can do about this because, depending on the installed version # of jquery.js, the tree widget breaks. persistent-cache-cpp-doc: embedded-javascript-library persistent-cache-cpp-1.0.5/debian/rules000077500000000000000000000003431414021052300200300ustar00rootroot00000000000000#!/usr/bin/make -f # -*- makefile -*- # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 export DPKG_GENSYMBOLS_CHECK_LEVEL=4 %: dh $@ --parallel --fail-missing override_dh_auto_configure: dh_auto_configure -- persistent-cache-cpp-1.0.5/debian/tests/000077500000000000000000000000001414021052300201125ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/debian/tests/control000066400000000000000000000001171414021052300215140ustar00rootroot00000000000000Tests: test-examples Depends: @, g++, libboost-dev, libleveldb-dev, pkg-config persistent-cache-cpp-1.0.5/debian/tests/test-examples000077500000000000000000000023631414021052300226370ustar00rootroot00000000000000#!/bin/sh # # Copyright (C) 2015 Canonical Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . # # Authored by: Michi Henning # set -e # Fail if any command fails. WORKDIR=$(mktemp -d) trap "rm -rf $WORKDIR" 0 INT QUIT ABRT PIPE TERM cd $WORKDIR cp /usr/share/doc/persistent-cache-cpp/examples/string_cache.cpp . cp /usr/share/doc/persistent-cache-cpp/examples/person_cache.cpp . g++ --std=c++11 -o string_cache string_cache.cpp `pkg-config --cflags --libs libpersistent-cache-cpp` [ -x string_cache ] g++ --std=c++11 -o person_cache person_cache.cpp `pkg-config --cflags --libs libpersistent-cache-cpp` [ -x person_cache ] ./string_cache ./person_cache echo "PASSED: autopkg tests" persistent-cache-cpp-1.0.5/doc/000077500000000000000000000000001414021052300162735ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/doc/CMakeLists.txt000066400000000000000000000000651414021052300210340ustar00rootroot00000000000000configure_file(main_page.dox.in main_page.dox @ONLY) persistent-cache-cpp-1.0.5/doc/main_page.dox.in000066400000000000000000000132461414021052300213420ustar00rootroot00000000000000/*! \mainpage \section overview Overview This API provides a persistent cache of key-value pairs. core::PersistentStringCache stores key, value, and metadata as type `std::string`. A simple type adapter template, core::PersistentCache, is provided to permit the use of types other than `string`. The cache is intended for caching arbitrary (possibly large) amounts of data, such as might be needed by a web browser cache. The cache scales to large numbers (millions) of entries and is [very fast](@ref performance). The implementation is based on [leveldb](http://leveldb.org) and typically provides throughput many times larger than the I/O bandwidth to disk. The cache is robust in the face of crashes and power loss. After a re-start, it is guaranteed to be in a consistent state with correct data. (Some number of updates that were made just prior to a power loss or kernel crash can be lost; however, if just the calling process crashes, no updates are lost.) A cache has a maximum size (which can be changed at any time). Once the cache reaches its maximum size, when adding an entry, the cache automatically discards enough entries to make room for the new entry. Keys can be (possibly binary) strings of size > 0. Values can be (possibly binary) strings including the empty string. Using the core::PersistentCache template, keys, values, and [metadata](@ref metadata) can be arbitrary user-defined types. The template requires the the application to provide encode and decode functions that convert each user-defined type to/from `string` and calls these functions automatically. This relieves the application from explicitly having to serialize or deserialize user-defined types when manipulating the cache. Entries maintain an access time, which is used to keep them in least-recently-used (LRU) order. In addition, entries can have an optional expiry time. (If no expiry time is specified, infinite expiry time is assumed.) \note The cache is thread-safe; you can call member functions from different threads without any synchronization. Thread-safety is provided for convenience, not performance. Calling concurrently into the cache from multiple threads will not yield improved performance. \subsection policy Discard policy The cache provides two different discard policies, `lru_ttl` and `lru_only`. For `lru_ttl`, the discard policy of the cache is to first delete all entries that have expired. If this does not free sufficient space to make room for a new entry, the cache then deletes entries in oldest to newest (LRU) order until sufficient space is available. This deletion in LRU order may delete entries that have an expiry time, but have not expired yet, as well as entries with infinite expiry time. For `lru_only`, entries do not maintain an expiry time and are therefore discarded strictly in LRU order. Access and expiry times are recorded with millisecond granularity. To indicate infinite expiry time, use the defaulted parameter value or `chrono::system_clock::time_point()`. \subsection metadata Metadata Besides storing key-value pairs, the cache allows you to add arbitrary extra data to each entry. This is useful, for example, to maintain metadata (such as HTTP header details) for the entries in the cache. \warning It is not possible to distinguish between "no metadata was added" and "empty metadata was added". Do not use the metadata in such a way that you rely the difference between "metadata not there" and "metadata is the empty string". \subsection errors Error reporting Methods throw `std::runtime_error` if the underlying database (leveldb) reports an error. If leveldb detects database corruption, the code throws `std::system_error` with with a 666 error code. To recover from this error, remove all files in the cache directory. Other errors are indicated by throwing `std::logic_error` or `std::invalid_argument` as appropriate. \subsection performance Performance Some rough performance figures, taken on an Intel Ivy Bridge i7-3770K 3.5 GHz with 16 GB RAM, appear below. Records are filled with random data to make them non-compressible. After filling the cache, the code performs cache lookups using random keys, with an 80% hit probability. On a miss, it inserts a new random record. This measures the typical steady-state behavior: whenever a cache miss happens, the caller fetches the data and inserts a new record into the cache. Setting | Value ---------- | ------ Cache size | 100 MB # Records | ~5100 Record size | 20 kB, normal distribution, stddev = 7000 Running the test With a 7200 rpm spinning disk produces: Parameter | Value -------------|------------ Reads | 30.9 MB/sec Writes | 7.0 MB/sec Records/sec | 1995 Running the test With an Intel 256 GB SSD produces: Parameter | Value -------------|------------ Reads | 80.4 MB/sec Writes | 15.7 MB/sec Records/sec | 4932 \note When benchmarking, make sure to compile in release mode. In debug mode, a number of expensive assertions are turned on. \note Also be aware that leveldb uses Snappy compression beneath the covers. This means that, if test data is simply filled with a fixed byte pattern, you will measure artificially high performance. \subsection linking Compiling and linking The API is provided as a static library, `lib@LIBNAME@.a`. (Code size on a 64-bit processor is less than 100 kB.) You can compile and link with the library as follows: g++ --std=c++11 -o myprog myprog.cpp `pkg-config --cflags --libs libpersistent-cache-cpp` -lleveldb \subsection examples Examples See core::PersistentStringCache and core::PersistentCache for simple usage examples. Additional code examples can be found in [@LIBNAME@/examples](file:///@CMAKE_INSTALL_PREFIX@/share/doc/@LIBNAME@/examples). */ persistent-cache-cpp-1.0.5/examples/000077500000000000000000000000001414021052300173445ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/examples/CMakeLists.txt000066400000000000000000000007451414021052300221120ustar00rootroot00000000000000set(install_dir ${CMAKE_INSTALL_PREFIX}/share/doc/${LIBNAME}/examples) set(examples string_cache person_cache ) foreach(e ${examples}) add_executable(${e} ${e}.cpp) target_link_libraries(${e} ${LIBNAME}) if ("${cmake_build_type_lower}" STREQUAL "coverage") SET_PROPERTY(TARGET ${e} APPEND_STRING PROPERTY LINK_FLAGS " -g --coverage") endif() install(FILES ${e}.cpp DESTINATION ${install_dir}) add_test(${e}_test ${e}) endforeach() persistent-cache-cpp-1.0.5/examples/README000066400000000000000000000004611414021052300202250ustar00rootroot00000000000000Examples in this directory: - string_cache Illustrates how to use the cache with keys, values, and metadata of type string. - person_cache Simple example to illustrate how to use the cache with a custom type Person. The important thing is to specialize the CacheCode template in namespace core. persistent-cache-cpp-1.0.5/examples/person_cache.cpp000066400000000000000000000036121414021052300225030ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #include #include using namespace std; struct Person { string name; int age; }; namespace core // Specializations must be placed into namespace core. { template <> string CacheCodec::encode(Person const& p) { ostringstream s; s << p.age << ' ' << p.name; return s.str(); } template <> Person CacheCodec::decode(string const& str) { istringstream s(str); Person p; s >> p.age >> p.name; return p; } } // namespace core #define DB_NAME "person_db" int main(int /* argc */, char** /* argv */) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" system("rm -fr " DB_NAME "/*"); #pragma GCC diagnostic pop using PersonCache = core::PersistentCache; auto c = PersonCache::open(DB_NAME, 1024 * 1024 * 1024, core::CacheDiscardPolicy::lru_only); Person bjarne{"Bjarne Stroustrup", 65}; c->put(bjarne, "C++ inventor"); auto value = c->get(bjarne); if (value) { cout << bjarne.name << ": " << *value << endl; } Person person{"no such person", 0}; value = c->get(person); assert(!value); } persistent-cache-cpp-1.0.5/examples/string_cache.cpp000066400000000000000000000052221414021052300225020ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #include #include using namespace std; #define DB_NAME "string_db" int main(int /* argc */, char** /* argv */) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" system("rm -fr " DB_NAME "/*"); #pragma GCC diagnostic pop auto c = core::PersistentStringCache::open(DB_NAME, 1024 * 1024 * 1024, core::CacheDiscardPolicy::lru_only); string const bjarne = "Bjarne Stroustrup"; // Retrieve a non-existent entry. auto value = c->get(bjarne); assert(!value); // Remove a non-existent entry. The return value indicates that it did not exist. assert(!c->invalidate(bjarne)); // Add an entry to the cache. c->put(bjarne, "C++ inventor"); // Retrieve the entry. value = c->get(bjarne); assert(value); assert(*value == "C++ inventor"); // Test that the entry is there. assert(c->contains_key(bjarne)); // Get metadata for the entry. assert(!c->get_metadata(bjarne)); // No metadata exists // Get value and metadata for the entry. auto data = c->get_data(bjarne); assert(data->value == "C++ inventor"); assert(data->metadata == ""); // No metadata exists // Add metadata. c->put_metadata(bjarne, "Born 30 December 1950"); // Check metadata. assert(*c->get_metadata(bjarne) == "Born 30 December 1950"); // It's not possible to delete metadata. The only thing we can do is // set it to the empty string. c->put_metadata(bjarne, ""); assert(*c->get_metadata(bjarne) == ""); // Remove the entry. The return value indicates that it existed and was removed. assert(c->invalidate(bjarne)); // Binary keys and values are fine. string bin_key = "a"; bin_key += '\0'; bin_key += "b"; string const bin_value(10, '\0'); c->put(bin_key, bin_value); data = c->get_data(bin_key); assert(data->value == bin_value); } persistent-cache-cpp-1.0.5/include/000077500000000000000000000000001414021052300171515ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/include/CMakeLists.txt000066400000000000000000000000271414021052300217100ustar00rootroot00000000000000add_subdirectory(core) persistent-cache-cpp-1.0.5/include/core/000077500000000000000000000000001414021052300201015ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/include/core/CMakeLists.txt000066400000000000000000000001041414021052300226340ustar00rootroot00000000000000file(GLOB hdrs *.h) install(FILES ${hdrs} DESTINATION include/core) persistent-cache-cpp-1.0.5/include/core/cache_codec.h000066400000000000000000000025261414021052300224570ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include namespace core { /** Traits for serialization and deserialization of cache custom types. To use custom types, specialize this template for each custom type (other than string) in the `core` namespace. \warning Do _not_ specialize this struct for `std::string`! Doing so has no effect. \see PersistentCache */ template struct CacheCodec { /** \brief Converts a value of custom type T into a string. */ static std::string encode(T const& value); /** \brief Converts a string into a value of custom type T. */ static T decode(std::string const& s); }; } // namespace core persistent-cache-cpp-1.0.5/include/core/cache_discard_policy.h000066400000000000000000000026111414021052300243650ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once namespace core { /** \brief Indicates the discard policy to make room for entries when the cache is full. Once the cache is full and another entry is added, `lru_ttl` unconditionally deletes all entries that have expired and then, if deleting these entries did not create sufficient free space, deletes entries in least-recently-used order until enough space is available. If the discard policy is set to `lru_only`, entries do not maintain an expiry time and are therefore discarded strictly in LRU order. */ enum class CacheDiscardPolicy { lru_ttl, ///< Evict expired entries first, followed by eviction in LRU order lru_only ///< Evict in LRU order }; } // namespace core persistent-cache-cpp-1.0.5/include/core/cache_events.h000066400000000000000000000064051414021052300227060ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include #include /** \brief Top-level namespace for core functionality. */ namespace core { /** \brief Event types that can be monitored. */ // Note: Any change here must have a corresponding change to // CacheEventIndex in core/internal/cache_event_indexes.h! enum class CacheEvent : uint32_t { get = 1 << 0, ///< An entry was returned by a call to `get()`, `get_or_put()`, `take()`, or `take_data()`. put = 1 << 1, ///< An entry was added by a call to `put()` or `get_or_put()`. invalidate = 1 << 2, ///< An entry was removed by a call to `invalidate()`, `take()`, or `take_data()`. touch = 1 << 3, ///< An entry was refreshed by a call to `touch()`. miss = 1 << 4, ///< A call to `get()`, `get_or_put()`, `take()`, or `take_data()` failed to return an entry. evict_ttl = 1 << 5, ///< An expired entry was evicted due to a call to `put()`, ///< `get_or_put()`, `trim_to()`, or `resize()`. evict_lru = 1 << 6, ///< The oldest entry was evicted due to a call to `put()`, ///< `get_or_put()`, `trim_to()`, or `resize()`. END_ = 1 << 7 ///< End marker }; /** \brief Returns the bitwise OR of two event types. */ inline CacheEvent operator|(CacheEvent left, CacheEvent right) { auto l = std::underlying_type::type(left); auto r = std::underlying_type::type(right); return CacheEvent(l | r); } /** \brief Assigns the bitwise OR of `left` and `right` to `left`. */ inline CacheEvent& operator|=(CacheEvent& left, CacheEvent right) { return left = left | right; } /** \brief Returns the bitwise AND of two event types. */ inline CacheEvent operator&(CacheEvent left, CacheEvent right) { auto l = std::underlying_type::type(left); auto r = std::underlying_type::type(right); return CacheEvent(l & r); } /** \brief Assigns the bitwise AND of `left` and `right` to `left`. */ inline CacheEvent& operator&=(CacheEvent& left, CacheEvent right) { return left = left & right; } /** \brief Returns the bitwise NOT of `ev`. Unused bits are set to zero. */ inline CacheEvent operator~(CacheEvent ev) { auto mask = std::underlying_type::type(CacheEvent::END_) - 1; auto event = std::underlying_type::type(ev); return CacheEvent(~event & mask); } /** \brief Convenience definition for all event types. */ static constexpr auto AllCacheEvents = CacheEvent(std::underlying_type::type(CacheEvent::END_) - 1); } // namespace core persistent-cache-cpp-1.0.5/include/core/internal/000077500000000000000000000000001414021052300217155ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/include/core/internal/cache_event_indexes.h000066400000000000000000000021671414021052300260570ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include namespace core { namespace internal { // Note: Any change here must have a corresponding change to // CacheEvent in core/cache_events.h! enum class CacheEventIndex : uint32_t { get = 0, put = 1, invalidate = 2, touch = 3, miss = 4, evict_ttl = 5, evict_lru = 6, END_ = 7 }; } // namespace internal } // namespace core persistent-cache-cpp-1.0.5/include/core/internal/persistent_string_cache_impl.h000066400000000000000000000164211414021052300300240ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include #include #include #include #include namespace core { namespace internal { class PersistentStringCacheStats; class PersistentStringCacheImpl { public: PersistentStringCacheImpl(std::string const& cache_path, int64_t max_size_in_bytes, core::CacheDiscardPolicy policy, PersistentStringCache* pimpl = nullptr); PersistentStringCacheImpl(std::string const& cache_path, PersistentStringCache* pimpl = nullptr); PersistentStringCacheImpl(PersistentStringCacheImpl const&) = delete; PersistentStringCacheImpl& operator=(PersistentStringCacheImpl const&) = delete; PersistentStringCacheImpl(PersistentStringCacheImpl&&) = delete; PersistentStringCacheImpl& operator=(PersistentStringCacheImpl&&) = delete; ~PersistentStringCacheImpl(); bool get(std::string const& key, std::string& value) const; bool get(std::string const& key, std::string& value, std::string* metadata) const; bool get_metadata(std::string const& key, std::string& metadata) const; bool contains_key(std::string const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; core::PersistentCacheStats stats() const; bool put(std::string const& key, std::string const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, char const* value, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, std::string const& value, std::string const* metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, char const* value_data, int64_t value_size, char const* metadata_data, int64_t metadata_size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool get_or_put(std::string const& key, std::string& value, PersistentStringCache::Loader load_func); bool get_or_put(std::string const& key, std::string& value, std::string* metadata, PersistentStringCache::Loader load_func); bool put_metadata(std::string const& key, std::string const& metadata); bool put_metadata(std::string const& key, char const* metadata, int64_t metadata_size); bool take(std::string const& key, std::string& value); bool take(std::string const& key, std::string& value, std::string* metadata); bool invalidate(std::string const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( std::string const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats() noexcept; void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); void set_handler(CacheEvent events, PersistentStringCache::EventCallback cb); private: // Simple struct to serialize/deserialize a data tuple. // For the stringified representation, fields are separated // by a space. struct DataTuple { int64_t atime; // Last access time, msec since the epoch int64_t etime; // Expiry time, msec since the epoch int64_t size; // Size in bytes DataTuple(int64_t at, int64_t et, int64_t s) noexcept : atime(at) , etime(et) , size(s) { } DataTuple() noexcept : DataTuple(0, 0, 0) { } DataTuple(std::string const& s) noexcept { std::istringstream is(s); is >> atime >> etime >> size; assert(!is.bad()); } DataTuple(DataTuple const&) = default; DataTuple(DataTuple&&) = default; DataTuple& operator=(DataTuple const&) = default; DataTuple& operator=(DataTuple&&) = default; std::string to_string() const { std::ostringstream os; os << atime << " " << etime << " " << size; return os.str(); } }; void init_stats(); void init_db(leveldb::Options options); bool cache_is_new() const; void write_version(); void check_version(); void read_settings(); void write_settings(); void read_stats(); void write_stats(); bool read_dirty_flag() const; void write_dirty_flag(bool is_dirty); DataTuple get_data(std::string const& key, bool& found) const; bool get_value_and_metadata(std::string const& key, DataTuple& data, std::string& value, std::string* metadata) const; void batch_delete(std::string const& key, DataTuple const& data, leveldb::WriteBatch& batch); void delete_entry(std::string const& key, DataTuple const& data); void delete_at_least(int64_t bytes_needed, std::string const& skip_key = ""); void call_handler(std::string const& key, core::internal::CacheEventIndex event) const; std::string make_message(leveldb::Status const& s, std::string const& msg) const; std::string make_message(std::string const& msg) const; void throw_if_error(leveldb::Status const& s, std::string const& msg) const; void throw_logic_error(std::string const& msg) const; void throw_invalid_argument(std::string const& msg) const; void throw_corrupt_error(std::string const& msg) const; PersistentStringCache* pimpl_; // Back-pointer to owning pimpl. std::unique_ptr block_cache_; // Must be defined *before* db_! std::unique_ptr db_; std::shared_ptr stats_; std::array(CacheEventIndex::END_)> handlers_; mutable std::recursive_mutex mutex_; }; } // namespace internal } // namespace core persistent-cache-cpp-1.0.5/include/core/internal/persistent_string_cache_stats.h000066400000000000000000000171261414021052300302240ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include #include #include #include #include #include namespace core { namespace internal { // Simple stats class to keep track of accesses. class PersistentStringCacheStats { public: PersistentStringCacheStats() noexcept : policy_(CacheDiscardPolicy::lru_only) , max_cache_size_(0) , num_entries_(0) , cache_size_(0) { clear(); hist_.resize(PersistentCacheStats::NUM_BINS, 0); } PersistentStringCacheStats(PersistentStringCacheStats const&) = default; PersistentStringCacheStats(PersistentStringCacheStats&&) = default; PersistentStringCacheStats& operator=(PersistentStringCacheStats const&) = default; PersistentStringCacheStats& operator=(PersistentStringCacheStats&&) = default; std::string cache_path_; // Immutable core::CacheDiscardPolicy policy_; // Immutable int64_t max_cache_size_; int64_t num_entries_; int64_t cache_size_; PersistentCacheStats::Histogram hist_; // Values below are reset by a call to clear(). int64_t hits_; int64_t misses_; int64_t hits_since_last_miss_; int64_t misses_since_last_hit_; int64_t longest_hit_run_; int64_t longest_miss_run_; int64_t num_hit_runs_; int64_t num_miss_runs_; int64_t ttl_evictions_; int64_t lru_evictions_; std::chrono::system_clock::time_point most_recent_hit_time_; std::chrono::system_clock::time_point most_recent_miss_time_; std::chrono::system_clock::time_point longest_hit_run_time_; std::chrono::system_clock::time_point longest_miss_run_time_; enum State { Initialized, LastAccessWasHit, LastAccessWasMiss }; State state_; void inc_hits() noexcept { ++hits_since_last_miss_; ++hits_; most_recent_hit_time_ = std::chrono::system_clock::now(); if (state_ != LastAccessWasHit) { ++num_hit_runs_; state_ = LastAccessWasHit; misses_since_last_hit_ = 0; } if (state_ != LastAccessWasMiss && hits_since_last_miss_ > longest_hit_run_) { longest_hit_run_ = hits_since_last_miss_; longest_hit_run_time_ = most_recent_hit_time_; } } void inc_misses() noexcept { ++misses_since_last_hit_; ++misses_; most_recent_miss_time_ = std::chrono::system_clock::now(); if (state_ != LastAccessWasMiss) { ++num_miss_runs_; state_ = LastAccessWasMiss; hits_since_last_miss_ = 0; } if (state_ != LastAccessWasHit && misses_since_last_hit_ > longest_miss_run_) { longest_miss_run_ = misses_since_last_hit_; longest_miss_run_time_ = most_recent_miss_time_; } } void hist_decrement(int64_t size) noexcept { assert(size > 0); --hist_[size_to_index(size)]; } void hist_increment(int64_t size) noexcept { assert(size > 0); ++hist_[size_to_index(size)]; } void hist_clear() noexcept { memset(&hist_[0], 0, hist_.size() * sizeof(PersistentCacheStats::Histogram::value_type)); } void clear() noexcept { state_ = Initialized; hits_ = 0; misses_ = 0; hits_since_last_miss_ = 0; misses_since_last_hit_ = 0; longest_hit_run_ = 0; longest_miss_run_ = 0; num_hit_runs_ = 0; num_miss_runs_ = 0; ttl_evictions_ = 0; lru_evictions_ = 0; most_recent_hit_time_ = std::chrono::system_clock::time_point(); most_recent_miss_time_ = std::chrono::system_clock::time_point(); longest_hit_run_time_ = std::chrono::system_clock::time_point(); longest_miss_run_time_ = std::chrono::system_clock::time_point(); } // Serialize the stats. std::string serialize() const { using namespace std; using namespace std::chrono; ostringstream os; os << state_ << " " << num_entries_ << " " << cache_size_ << " " << hits_ << " " << misses_ << " " << hits_since_last_miss_ << " " << misses_since_last_hit_ << " " << longest_hit_run_ << " " << longest_miss_run_ << " " << num_hit_runs_ << " " << num_miss_runs_ << " " << ttl_evictions_ << " " << lru_evictions_ << " " << duration_cast(most_recent_hit_time_.time_since_epoch()).count() << " " << duration_cast(most_recent_miss_time_.time_since_epoch()).count() << " " << duration_cast(longest_hit_run_time_.time_since_epoch()).count() << " " << duration_cast(longest_miss_run_time_.time_since_epoch()).count(); for (auto d : hist_) { os << " " << d; } return os.str(); } // De-serialize the stats. void deserialize(const std::string& s) noexcept { using namespace std; using namespace std::chrono; istringstream is(s); int64_t state; int64_t mrht; int64_t mrmt; int64_t lhrt; int64_t lmrt; is >> state >> num_entries_ >> cache_size_ >> hits_ >> misses_ >> hits_since_last_miss_ >> misses_since_last_hit_ >> longest_hit_run_ >> longest_miss_run_ >> num_hit_runs_ >> num_miss_runs_ >> ttl_evictions_ >> lru_evictions_ >> mrht >> mrmt >> lhrt >> lmrt; for (unsigned i = 0; i < PersistentCacheStats::NUM_BINS; ++i) { is >> hist_[i]; } assert(!is.bad()); state_ = static_cast(state); most_recent_hit_time_ = system_clock::time_point(milliseconds(mrht)); most_recent_miss_time_ = system_clock::time_point(milliseconds(mrmt)); longest_hit_run_time_ = system_clock::time_point(milliseconds(lhrt)); longest_miss_run_time_ = system_clock::time_point(milliseconds(lmrt)); } private: unsigned size_to_index(int64_t size) const noexcept { using namespace std; assert(size > 0); unsigned log = floor(log10(size)); // 0..9 = 0, 10..99 = 1, 100..199 = 2, etc. unsigned exp = pow(10, log); // 0..9 = 1, 10..99 = 10, 100..199 = 100, etc. unsigned div = size / exp; // Extracts first decimal digit of size. int index = log * 10 + div - log - 1; // Partition each power of 10 into 9 bins. index -= 8; // Sizes < 10 all go into bin 0; return index < 0 ? 0 : (index > int(hist_.size()) - 1 ? hist_.size() - 1 : index); } }; } // namespace internal } // namespace core persistent-cache-cpp-1.0.5/include/core/optional.h000066400000000000000000000020421414021052300220750ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include namespace core { /** \brief Convenience typedef for nullable values. \note You should use `core::Optional` in preference to `boost::optional` in your code. This will ease an eventual transition to `std::optional`. */ template using Optional = boost::optional; } // namespace core persistent-cache-cpp-1.0.5/include/core/persistent_cache.h000066400000000000000000003134261414021052300236060ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include #include namespace core { /** \brief A persistent cache of key-value pairs and metadata of user-defined type. `K`, `V`, and `M` are the key type, value type, and metadata type, respectively. \note This template is a simple type adapter that forwards to core::PersistentStringCache. See the documentation there for details on cache operations and semantics. In order to use the cache with custom types (other than `std::string`), you must provide methods to encode the type to `string`, and decode from `string` back to the type. For example, suppose we have the following structure that we want to use as the key type of the cache: \code{.cpp} struct Person { string name; int age; }; \endcode In order to use the cache with the `Person` struct as the key, you must specialize the CacheCodec struct in namespace `core`: \code{.cpp} namespace core // Specializations must be placed into namespace core. { template <> string CacheCodec::encode(Person const& p) { ostringstream s; s << p.age << ' ' << p.name; return s.str(); } template <> Person CacheCodec::decode(string const& str) { istringstream s; Person p; s >> p.age >> p.name; return p; } } // namespace core \endcode For this example, it is convenient to stream the age first because this guarantees that `decode()` will work correctly even if the name contains a space. The order in which you stream the fields does not matter, only that (for custom _key_ types) the string representation of each value is unique. With these two methods defined, we can now use the cache with `Person` instances as the key. For example: \code{.cpp} // Custom cache using Person as the key, and string as the value and metadata. using PersonCache = core::PersistentCache; auto c = PersonCache::open("my_cache", 1024 * 1024 * 1024, CacheDiscardPolicy::LRU_only); Person bjarne{"Bjarne Stroustrup", 65}; c->put(bjarne, "C++ inventor"); auto value = c->get(bjarne); if (value) { cout << bjarne.name << ": " << *value << endl; } Person person{"no such person", 0}; value = c->get(person); assert(!value); \endcode Running this code produces the output: \code Bjarne Stroustrup: C++ inventor \endcode You can use a custom type for the cache's value and metadata as well by simply providing CacheCodec specializations as needed. \see core::CacheCodec \see core::PersistentStringCache */ template class PersistentCache { public: /** Convenience typedef for the return type of open(). */ typedef std::unique_ptr> UPtr; /** \brief Simple pair of value and metadata. */ struct Data { /** \brief Stores the value of an entry. */ V value; /** \brief Stores the metadata of an entry. If no metadata exists for an entry, `metadata` is returned as the empty string when it is retrieved. */ M metadata; }; /** @name Typedefs for nullable keys, values, and metadata. */ //{@ /** \brief Convenience typedefs for returning nullable values. \note You should use `OptionalKey`, `OptionalValue`, and `OptionalMetadata` in your code in preference to `boost::optional`. This will ease an eventual transition to `std::optional`. */ typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; typedef Optional OptionalData; //@} /** @name Copy and Assignment */ //{@ PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; //@} /** Destroys the instance. The destructor compacts the database. This ensures that, while a cache is not in use, it comsumes as little disk space as possible. */ ~PersistentCache() = default; /** @name Creation Methods */ //{@ /** \brief Creates or opens a PersistentCache. */ static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); /** \brief Opens an existing PersistentCache. */ static UPtr open(std::string const& cache_path); //@} /** @name Accessors */ //{@ /** \brief Returns the value of an entry in the cache, provided the entry has not expired. */ OptionalValue get(K const& key) const; /** \brief Returns the data for an entry in the cache, provided the entry has not expired. */ OptionalData get_data(K const& key) const; /** \brief Returns the metadata for an entry in the cache, provided the entry has not expired. */ OptionalMetadata get_metadata(K const& key) const; /** \brief Tests if an (unexpired) entry is in the cache. */ bool contains_key(K const& key) const; /** \brief Returns the number of entries in the cache. */ int64_t size() const noexcept; /** \brief Returns the number of bytes consumed by entries in the cache. */ int64_t size_in_bytes() const noexcept; /** \brief Returns the maximum size of the cache in bytes. */ int64_t max_size_in_bytes() const noexcept; /** \brief Returns an estimate of the disk space consumed by the cache. */ int64_t disk_size_in_bytes() const; /** \brief Returns the discard policy of the cache. */ CacheDiscardPolicy discard_policy() const noexcept; /** \brief Returns statistics for the cache. */ PersistentCacheStats stats() const; //@} /** @name Modifiers */ //{@ /** \brief Adds or updates an entry. If `V` = `std::string`, the method is also overloaded to to accept `char const*` and `size`. */ bool put(K const& key, V const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Adds or updates an entry and its metadata. If 'V' or `M` = `std::string`, the method is also overloaded to accept `char const*` and `size`. */ bool put(K const& key, V const& value, M const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Function called by the cache to load an entry after a cache miss. */ typedef std::function& cache)> Loader; /** \brief Atomically retrieves or stores a cache entry. */ OptionalValue get_or_put(K const& key, Loader const& load_func); /** \brief Atomically retrieves or stores a cache entry. */ OptionalData get_or_put_data(K const& key, Loader const& load_func); /** \brief Adds or replaces the metadata for an entry. If `M` = `std::string`, an overload that accepts `const char*` and `size` is provided as well. */ bool put_metadata(K const& key, M const& metadata); /** \brief Removes an entry and returns its value. */ OptionalValue take(K const& key); /** \brief Removes an entry and returns its value and metadata. */ OptionalData take_data(K const& key); /** \brief Removes an entry and its associated metadata (if any). */ bool invalidate(K const& key); /** \brief Atomically removes the specified entries from the cache. */ void invalidate(std::vector const& keys); /** \brief Atomically removes the specified entries from the cache. */ template void invalidate(It begin, It end); /** \brief Atomically removes the specified entries from the cache. */ void invalidate(std::initializer_list const& keys); /** \brief Deletes all entries from the cache. */ void invalidate(); /** \brief Updates the access time of an entry. */ bool touch( K const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Resets all statistics counters. */ void clear_stats(); /** \brief Changes the maximum size of the cache. */ void resize(int64_t size_in_bytes); /** \brief Expires entries. */ void trim_to(int64_t used_size_in_bytes); /** \brief Compacts the database. */ void compact(); //@} /** @name Monitoring cache activity */ //{@ /** \brief The type of a handler function. */ typedef std::function EventCallback; /** \brief Installs a handler for one or more events. */ void set_handler(CacheEvent events, EventCallback cb); //@} private: // @cond PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; // @endcond }; // @cond // Method implementations. Out-of-line because, otherwise, things become completely unreadable. template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr(new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open(std::string const& cache_path) { return PersistentCache::UPtr(new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get(K const& key) const { auto const& svalue = p_->get(CacheCodec::encode(key)); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data(K const& key) const { auto sdata = p_->get_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), CacheCodec::decode(sdata->metadata)}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata(K const& key) const { auto smeta = p_->get_metadata(CacheCodec::encode(key)); return smeta ? OptionalMetadata(CacheCodec::decode(*smeta)) : OptionalMetadata(); } template bool PersistentCache::contains_key(K const& key) const { return p_->contains_key(CacheCodec::encode(key)); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(K const& key, V const& value, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), CacheCodec::encode(value), expiry_time); } template bool PersistentCache::put(K const& key, V const& value, M const& metadata, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), CacheCodec::encode(value), CacheCodec::encode(metadata), expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( K const& key, PersistentCache::Loader const& load_func) { std::string const& skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(skey, sload_func); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( K const& key, PersistentCache::Loader const& load_func) { std::string const& skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(skey, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::put_metadata(K const& key, M const& metadata) { return p_->put_metadata(CacheCodec::encode(key), CacheCodec::encode(metadata)); } template typename PersistentCache::OptionalValue PersistentCache::take(K const& key) { auto svalue = p_->take(CacheCodec::encode(key)); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data(K const& key) { auto sdata = p_->take_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::invalidate(K const& key) { return p_->invalidate(CacheCodec::encode(key)); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(CacheCodec::encode(*it)); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(K const& key, std::chrono::time_point expiry_time) { return p_->touch(CacheCodec::encode(key), expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { auto scb = [cb](std::string const& key, CacheEvent ev, PersistentCacheStats const& c) { cb(CacheCodec::decode(key), ev, c); }; p_->set_handler(events, scb); } // Below are specializations for the various combinations of one or more of K, V, and M // being of type std::string. Without this, we would end up calling through a string-to-string // codec, which would force a copy for everything of type string, which is brutally inefficient. // // This is verbose because we must specialize the class template in order to avoid // calling the CacheCodec methods for type string. The combinations that follow are // , , , , , , and . // The code is identical, except that, where we know that K, V, or M are std::string, // it avoids calling the encode()/decode() methods. // Specialization for K = std::string. template class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { V value; M metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(std::string const& key) const; OptionalData get_data(std::string const& key) const; OptionalMetadata get_metadata(std::string const& key) const; bool contains_key(std::string const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(std::string const& key, V const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, V const& value, M const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(std::string const& key, Loader const& load_func); OptionalData get_or_put_data(std::string const& key, Loader const& load_func); bool put_metadata(std::string const& key, M const& metadata); OptionalValue take(std::string const& key); OptionalData take_data(std::string const& key); bool invalidate(std::string const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( std::string const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path) { return PersistentCache::UPtr(new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get( std::string const& key) const { auto const& svalue = p_->get(key); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data( std::string const& key) const { auto sdata = p_->get_data(key); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), CacheCodec::decode(sdata->metadata)}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata( std::string const& key) const { auto smeta = p_->get_metadata(key); return smeta ? OptionalMetadata(CacheCodec::decode(*smeta)) : OptionalMetadata(); } template bool PersistentCache::contains_key(std::string const& key) const { return p_->contains_key(key); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(std::string const& key, V const& value, std::chrono::time_point expiry_time) { return p_->put(key, CacheCodec::encode(value), expiry_time); } template bool PersistentCache::put(std::string const& key, V const& value, M const& metadata, std::chrono::time_point expiry_time) { return p_->put(key, CacheCodec::encode(value), CacheCodec::encode(metadata), expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(key, sload_func); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(key, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::put_metadata(std::string const& key, M const& metadata) { return p_->put_metadata(key, CacheCodec::encode(metadata)); } template typename PersistentCache::OptionalValue PersistentCache::take( std::string const& key) { auto svalue = p_->take(key); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data( std::string const& key) { auto sdata = p_->take_data(key); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::invalidate(std::string const& key) { return p_->invalidate(key); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(*it); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(std::string const& key, std::chrono::time_point expiry_time) { return p_->touch(key, expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { p_->set_handler(events, cb); } // Specialization for V = std::string. template class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { std::string value; M metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(K const& key) const; OptionalData get_data(K const& key) const; OptionalMetadata get_metadata(K const& key) const; bool contains_key(K const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(K const& key, std::string const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, char const* value, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, std::string const& value, M const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, char const* value, int64_t value_size, M const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(K const& key, Loader const& load_func); OptionalData get_or_put_data(K const& key, Loader const& load_func); bool put_metadata(K const& key, M const& metadata); OptionalValue take(K const& key); OptionalData take_data(K const& key); bool invalidate(K const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( K const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path) { return PersistentCache::UPtr(new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get(K const& key) const { auto const& svalue = p_->get(CacheCodec::encode(key)); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data( K const& key) const { auto sdata = p_->get_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, CacheCodec::decode(sdata->metadata)}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata( K const& key) const { auto smeta = p_->get_metadata(CacheCodec::encode(key)); return smeta ? OptionalMetadata(CacheCodec::decode(*smeta)) : OptionalMetadata(); } template bool PersistentCache::contains_key(K const& key) const { return p_->contains_key(CacheCodec::encode(key)); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(K const& key, std::string const& value, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, expiry_time); } template bool PersistentCache::put(K const& key, char const* value, int64_t size, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, size, expiry_time); } template bool PersistentCache::put(K const& key, std::string const& value, M const& metadata, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, CacheCodec::encode(metadata), expiry_time); } template bool PersistentCache::put(K const& key, char const* value, int64_t size, M const& metadata, std::chrono::time_point expiry_time) { std::string md = CacheCodec::encode(metadata); return p_->put(CacheCodec::encode(key), value, size, md.data(), md.size(), expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( K const& key, PersistentCache::Loader const& load_func) { std::string const& skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(skey, sload_func); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( K const& key, PersistentCache::Loader const& load_func) { std::string const& skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(skey, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::put_metadata(K const& key, M const& metadata) { return p_->put_metadata(CacheCodec::encode(key), CacheCodec::encode(metadata)); } template typename PersistentCache::OptionalValue PersistentCache::take(K const& key) { auto svalue = p_->take(CacheCodec::encode(key)); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data( K const& key) { auto sdata = p_->take_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::invalidate(K const& key) { return p_->invalidate(CacheCodec::encode(key)); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(CacheCodec::encode(*it)); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(K const& key, std::chrono::time_point expiry_time) { return p_->touch(CacheCodec::encode(key), expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { auto scb = [cb](std::string const& key, CacheEvent ev, PersistentCacheStats const& c) { cb(CacheCodec::decode(key), ev, c); }; p_->set_handler(events, scb); } // Specialization for M = std::string. template class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { V value; std::string metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(K const& key) const; OptionalData get_data(K const& key) const; OptionalMetadata get_metadata(K const& key) const; bool contains_key(K const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(K const& key, V const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, V const& value, std::string const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, V const& value, char const* metadata, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(K const& key, Loader const& load_func); OptionalData get_or_put_data(K const& key, Loader const& load_func); bool put_metadata(K const& key, std::string const& metadata); bool put_metadata(K const& key, char const* metadata, int64_t size); OptionalValue take(K const& key); OptionalData take_data(K const& key); bool invalidate(K const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( K const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path) { return PersistentCache::UPtr(new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get(K const& key) const { auto const& svalue = p_->get(CacheCodec::encode(key)); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data( K const& key) const { auto sdata = p_->get_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), sdata->metadata}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata( K const& key) const { auto smeta = p_->get_metadata(CacheCodec::encode(key)); return smeta ? OptionalMetadata(*smeta) : OptionalMetadata(); } template bool PersistentCache::contains_key(K const& key) const { return p_->contains_key(CacheCodec::encode(key)); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(K const& key, V const& value, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), CacheCodec::encode(value), expiry_time); } template bool PersistentCache::put(K const& key, V const& value, std::string const& metadata, std::chrono::time_point expiry_time) { std::string v = CacheCodec::encode(value); return p_->put(CacheCodec::encode(key), v.data(), v.size(), metadata.data(), metadata.size(), expiry_time); } template bool PersistentCache::put(K const& key, V const& value, char const* metadata, int64_t size, std::chrono::time_point expiry_time) { std::string v = CacheCodec::encode(value); return p_->put(CacheCodec::encode(key), v.data(), v.size(), metadata, size, expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( K const& key, PersistentCache::Loader const& load_func) { std::string const& skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(skey, sload_func); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( K const& key, PersistentCache::Loader const& load_func) { std::string const& skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(skey, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), sdata->metadata}); } template bool PersistentCache::put_metadata(K const& key, std::string const& metadata) { return p_->put_metadata(CacheCodec::encode(key), metadata); } template bool PersistentCache::put_metadata(K const& key, char const* metadata, int64_t size) { return p_->put_metadata(CacheCodec::encode(key), metadata, size); } template typename PersistentCache::OptionalValue PersistentCache::take(K const& key) { auto svalue = p_->take(CacheCodec::encode(key)); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data( K const& key) { auto sdata = p_->take_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), sdata->metadata}); } template bool PersistentCache::invalidate(K const& key) { return p_->invalidate(CacheCodec::encode(key)); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(CacheCodec::encode(*it)); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(K const& key, std::chrono::time_point expiry_time) { return p_->touch(CacheCodec::encode(key), expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { auto scb = [cb](std::string const& key, CacheEvent ev, PersistentCacheStats const& c) { cb(CacheCodec::decode(key), ev, c); }; p_->set_handler(events, scb); } // Specialization for K and V = std::string. template class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { std::string value; M metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(std::string const& key) const; OptionalData get_data(std::string const& key) const; OptionalMetadata get_metadata(std::string const& key) const; bool contains_key(std::string const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(std::string const& key, std::string const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, char const* value, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, std::string const& value, M const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, char const* value, int64_t value_size, M const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(std::string const& key, Loader const& load_func); OptionalData get_or_put_data(std::string const& key, Loader const& load_func); bool put_metadata(std::string const& key, M const& metadata); OptionalValue take(std::string const& key); OptionalData take_data(std::string const& key); bool invalidate(std::string const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( std::string const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path) { return PersistentCache::UPtr( new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get( std::string const& key) const { auto const& svalue = p_->get(key); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data(std::string const& key) const { auto sdata = p_->get_data(key); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, CacheCodec::decode(sdata->metadata)}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata(std::string const& key) const { auto smeta = p_->get_metadata(key); return smeta ? OptionalMetadata(CacheCodec::decode(*smeta)) : OptionalMetadata(); } template bool PersistentCache::contains_key(std::string const& key) const { return p_->contains_key(key); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(std::string const& key, std::string const& value, std::chrono::time_point expiry_time) { return p_->put(key, value, expiry_time); } template bool PersistentCache::put(std::string const& key, char const* value, int64_t size, std::chrono::time_point expiry_time) { return p_->put(key, value, size, nullptr, 0, expiry_time); } template bool PersistentCache::put(std::string const& key, std::string const& value, M const& metadata, std::chrono::time_point expiry_time) { return p_->put(key, value, CacheCodec::encode(metadata), expiry_time); } template bool PersistentCache::put(std::string const& key, char const* value, int64_t size, M const& metadata, std::chrono::time_point expiry_time) { std::string md = CacheCodec::encode(metadata); return p_->put(key, value, size, md.data(), md.size(), expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(key, sload_func); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(key, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::put_metadata(std::string const& key, M const& metadata) { return p_->put_metadata(key, CacheCodec::encode(metadata)); } template typename PersistentCache::OptionalValue PersistentCache::take( std::string const& key) { auto svalue = p_->take(key); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data(std::string const& key) { auto sdata = p_->take_data(key); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, CacheCodec::decode(sdata->metadata)}); } template bool PersistentCache::invalidate(std::string const& key) { return p_->invalidate(key); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(*it); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(std::string const& key, std::chrono::time_point expiry_time) { return p_->touch(key, expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { p_->set_handler(events, cb); } // Specialization for K and M = std::string. template class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { V value; std::string metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(std::string const& key) const; OptionalData get_data(std::string const& key) const; OptionalMetadata get_metadata(std::string const& key) const; bool contains_key(std::string const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(std::string const& key, V const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, V const& value, std::string const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, V const& value, char const* metadata, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(std::string const& key, Loader const& load_func); OptionalData get_or_put_data(std::string const& key, Loader const& load_func); bool put_metadata(std::string const& key, std::string const& metadata); bool put_metadata(std::string const& key, char const* metadata, int64_t size); OptionalValue take(std::string const& key); OptionalData take_data(std::string const& key); bool invalidate(std::string const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( std::string const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path) { return PersistentCache::UPtr( new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get( std::string const& key) const { auto const& svalue = p_->get(key); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data(std::string const& key) const { auto sdata = p_->get_data(key); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), sdata->metadata}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata(std::string const& key) const { auto smeta = p_->get_metadata(key); return smeta ? OptionalMetadata(*smeta) : OptionalMetadata(); } template bool PersistentCache::contains_key(std::string const& key) const { return p_->contains_key(key); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(std::string const& key, V const& value, std::chrono::time_point expiry_time) { return p_->put(key, CacheCodec::encode(value), expiry_time); } template bool PersistentCache::put(std::string const& key, V const& value, std::string const& metadata, std::chrono::time_point expiry_time) { std::string v = CacheCodec::encode(value); return p_->put(key, v.data(), v.size(), metadata.data(), metadata.size(), expiry_time); } template bool PersistentCache::put(std::string const& key, V const& value, char const* metadata, int64_t size, std::chrono::time_point expiry_time) { std::string v = CacheCodec::encode(value); return p_->put(key, v.data(), v.size(), metadata, size, expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(key, sload_func); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(key, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), sdata->metadata}); } template bool PersistentCache::put_metadata(std::string const& key, std::string const& metadata) { return p_->put_metadata(key, metadata); } template bool PersistentCache::put_metadata(std::string const& key, char const* metadata, int64_t size) { return p_->put_metadata(key, metadata, size); } template typename PersistentCache::OptionalValue PersistentCache::take( std::string const& key) { auto svalue = p_->take(key); return svalue ? OptionalValue(CacheCodec::decode(*svalue)) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data(std::string const& key) { auto sdata = p_->take_data(key); if (!sdata) { return OptionalData(); } return OptionalData({CacheCodec::decode(sdata->value), sdata->metadata}); } template bool PersistentCache::invalidate(std::string const& key) { return p_->invalidate(key); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(*it); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(std::string const& key, std::chrono::time_point expiry_time) { return p_->touch(key, expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { p_->set_handler(events, cb); } // Specialization for V and M = std::string. template class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { std::string value; std::string metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(K const& key) const; OptionalData get_data(K const& key) const; OptionalMetadata get_metadata(K const& key) const; bool contains_key(K const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(K const& key, std::string const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, char const* value, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, std::string const& value, std::string const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(K const& key, char const* value, int64_t value_size, char const* metadata, int64_t metadata_size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(K const& key, Loader const& load_func); OptionalData get_or_put_data(K const& key, Loader const& load_func); bool put_metadata(K const& key, std::string const& metadata); bool put_metadata(K const& key, char const* metadata, int64_t size); OptionalValue take(K const& key); OptionalData take_data(K const& key); bool invalidate(K const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( K const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; template PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } template PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } template typename PersistentCache::UPtr PersistentCache::open( std::string const& cache_path) { return PersistentCache::UPtr( new PersistentCache(cache_path)); } template typename PersistentCache::OptionalValue PersistentCache::get( K const& key) const { auto const& svalue = p_->get(CacheCodec::encode(key)); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_data(K const& key) const { auto sdata = p_->get_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, sdata->metadata}); } template typename PersistentCache::OptionalMetadata PersistentCache::get_metadata(K const& key) const { auto smeta = p_->get_metadata(CacheCodec::encode(key)); return smeta ? OptionalMetadata(*smeta) : OptionalMetadata(); } template bool PersistentCache::contains_key(K const& key) const { return p_->contains_key(CacheCodec::encode(key)); } template int64_t PersistentCache::size() const noexcept { return p_->size(); } template int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } template int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } template int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } template CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } template PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } template bool PersistentCache::put(K const& key, std::string const& value, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, expiry_time); } template bool PersistentCache::put(K const& key, char const* value, int64_t size, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, size, expiry_time); } template bool PersistentCache::put(K const& key, std::string const& value, std::string const& metadata, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, metadata, expiry_time); } template bool PersistentCache::put(K const& key, char const* value, int64_t value_size, char const* metadata, int64_t metadata_size, std::chrono::time_point expiry_time) { return p_->put(CacheCodec::encode(key), value, value_size, metadata, metadata_size, expiry_time); } template typename PersistentCache::OptionalValue PersistentCache::get_or_put( K const& key, PersistentCache::Loader const& load_func) { auto skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(skey, sload_func); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::get_or_put_data( K const& key, PersistentCache::Loader const& load_func) { auto skey = CacheCodec::encode(key); auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(skey, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, sdata->metadata}); } template bool PersistentCache::put_metadata(K const& key, std::string const& metadata) { return p_->put_metadata(CacheCodec::encode(key), metadata); } template bool PersistentCache::put_metadata(K const& key, char const* metadata, int64_t size) { return p_->put_metadata(CacheCodec::encode(key), metadata, size); } template typename PersistentCache::OptionalValue PersistentCache::take( K const& key) { auto svalue = p_->take(CacheCodec::encode(key)); return svalue ? OptionalValue(*svalue) : OptionalValue(); } template typename PersistentCache::OptionalData PersistentCache::take_data(K const& key) { auto sdata = p_->take_data(CacheCodec::encode(key)); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, sdata->metadata}); } template bool PersistentCache::invalidate(K const& key) { return p_->invalidate(CacheCodec::encode(key)); } template void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(CacheCodec::encode(*it)); } p_->invalidate(skeys.begin(), skeys.end()); } template void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate() { p_->invalidate(); } template bool PersistentCache::touch(K const& key, std::chrono::time_point expiry_time) { return p_->touch(CacheCodec::encode(key), expiry_time); } template void PersistentCache::clear_stats() { p_->clear_stats(); } template void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } template void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } template void PersistentCache::compact() { p_->compact(); } template void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { auto scb = [cb](std::string const& key, CacheEvent ev, PersistentCacheStats const& c) { cb(CacheCodec::decode(key), ev, c); }; p_->set_handler(events, scb); } // Specialization for K, V, and M = std::string. template <> class PersistentCache { public: typedef std::unique_ptr> UPtr; typedef Optional OptionalKey; typedef Optional OptionalValue; typedef Optional OptionalMetadata; struct Data { std::string value; std::string metadata; }; typedef Optional OptionalData; PersistentCache(PersistentCache const&) = delete; PersistentCache& operator=(PersistentCache const&) = delete; PersistentCache(PersistentCache&&) = default; PersistentCache& operator=(PersistentCache&&) = default; ~PersistentCache() = default; static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); static UPtr open(std::string const& cache_path); OptionalValue get(std::string const& key) const; OptionalData get_data(std::string const& key) const; OptionalMetadata get_metadata(std::string const& key) const; bool contains_key(std::string const& key) const; int64_t size() const noexcept; int64_t size_in_bytes() const noexcept; int64_t max_size_in_bytes() const noexcept; int64_t disk_size_in_bytes() const; CacheDiscardPolicy discard_policy() const noexcept; PersistentCacheStats stats() const; bool put(std::string const& key, std::string const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, char const* value, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, std::string const& value, std::string const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); bool put(std::string const& key, char const* value, int64_t value_size, char const* metadata, int64_t metadata_size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); typedef std::function& cache)> Loader; OptionalValue get_or_put(std::string const& key, Loader const& load_func); OptionalData get_or_put_data(std::string const& key, Loader const& load_func); bool put_metadata(std::string const& key, std::string const& metadata); bool put_metadata(std::string const& key, char const* metadata, int64_t size); OptionalValue take(std::string const& key); OptionalData take_data(std::string const& key); bool invalidate(std::string const& key); void invalidate(std::vector const& keys); template void invalidate(It begin, It end); void invalidate(std::initializer_list const& keys); void invalidate(); bool touch( std::string const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); void clear_stats(); void resize(int64_t size_in_bytes); void trim_to(int64_t used_size_in_bytes); void compact(); typedef std::function EventCallback; void set_handler(CacheEvent events, EventCallback cb); private: PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentCache(std::string const& cache_path); std::unique_ptr p_; }; PersistentCache::PersistentCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(PersistentStringCache::open(cache_path, max_size_in_bytes, policy)) { } PersistentCache::PersistentCache(std::string const& cache_path) : p_(PersistentStringCache::open(cache_path)) { } typename PersistentCache::UPtr PersistentCache::open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentCache::UPtr( new PersistentCache(cache_path, max_size_in_bytes, policy)); } typename PersistentCache::UPtr PersistentCache::open(std::string const& cache_path) { return PersistentCache::UPtr( new PersistentCache(cache_path)); } typename PersistentCache::OptionalValue PersistentCache::get(std::string const& key) const { auto const& svalue = p_->get(key); return svalue ? OptionalValue(*svalue) : OptionalValue(); } typename PersistentCache::OptionalData PersistentCache::get_data(std::string const& key) const { auto sdata = p_->get_data(key); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, sdata->metadata}); } typename PersistentCache::OptionalMetadata PersistentCache::get_metadata(std::string const& key) const { auto smeta = p_->get_metadata(key); return smeta ? OptionalMetadata(*smeta) : OptionalMetadata(); } bool PersistentCache::contains_key(std::string const& key) const { return p_->contains_key(key); } int64_t PersistentCache::size() const noexcept { return p_->size(); } int64_t PersistentCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } int64_t PersistentCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } int64_t PersistentCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } CacheDiscardPolicy PersistentCache::discard_policy() const noexcept { return p_->discard_policy(); } PersistentCacheStats PersistentCache::stats() const { return p_->stats(); } bool PersistentCache::put( std::string const& key, std::string const& value, std::chrono::time_point expiry_time) { return p_->put(key, value, expiry_time); } bool PersistentCache::put( std::string const& key, char const* value, int64_t size, std::chrono::time_point expiry_time) { return p_->put(key, value, size, expiry_time); } bool PersistentCache::put( std::string const& key, std::string const& value, std::string const& metadata, std::chrono::time_point expiry_time) { return p_->put(key, value, metadata, expiry_time); } bool PersistentCache::put( std::string const& key, char const* value, int64_t value_size, char const* metadata, int64_t metadata_size, std::chrono::time_point expiry_time) { return p_->put(key, value, value_size, metadata, metadata_size, expiry_time); } typename PersistentCache::OptionalValue PersistentCache::get_or_put( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto svalue = p_->get_or_put(key, sload_func); return svalue ? OptionalValue(*svalue) : OptionalValue(); } typename PersistentCache::OptionalData PersistentCache::get_or_put_data( std::string const& key, PersistentCache::Loader const& load_func) { auto sload_func = [&](std::string const&, PersistentStringCache const&) { load_func(key, *this); }; auto sdata = p_->get_or_put_data(key, sload_func); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, sdata->metadata}); } bool PersistentCache::put_metadata(std::string const& key, std::string const& metadata) { return p_->put_metadata(key, metadata); } bool PersistentCache::put_metadata(std::string const& key, char const* metadata, int64_t size) { return p_->put_metadata(key, metadata, size); } typename PersistentCache::OptionalValue PersistentCache::take(std::string const& key) { auto svalue = p_->take(key); return svalue ? OptionalValue(*svalue) : OptionalValue(); } typename PersistentCache::OptionalData PersistentCache::take_data(std::string const& key) { auto sdata = p_->take_data(key); if (!sdata) { return OptionalData(); } return OptionalData({sdata->value, sdata->metadata}); } bool PersistentCache::invalidate(std::string const& key) { return p_->invalidate(key); } void PersistentCache::invalidate(std::vector const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentCache::invalidate(It begin, It end) { std::vector skeys; for (auto&& it = begin; it < end; ++it) { skeys.push_back(*it); } p_->invalidate(skeys.begin(), skeys.end()); } void PersistentCache::invalidate(std::initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } void PersistentCache::invalidate() { p_->invalidate(); } bool PersistentCache::touch( std::string const& key, std::chrono::time_point expiry_time) { return p_->touch(key, expiry_time); } void PersistentCache::clear_stats() { p_->clear_stats(); } void PersistentCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } void PersistentCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } void PersistentCache::compact() { p_->compact(); } void PersistentCache::set_handler(CacheEvent events, EventCallback cb) { p_->set_handler(events, cb); } // @endcond } // namespace core persistent-cache-cpp-1.0.5/include/core/persistent_cache_stats.h000066400000000000000000000157001414021052300250160ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include #include #include #include namespace core { namespace internal { class PersistentStringCacheImpl; class PersistentStringCacheStats; } // namespace internal /** \brief Class that provides (read-only) access to cache statistics and settings. */ class PersistentCacheStats { public: /** @name Construction, Copy and Assignment Copy and assignment have the usual value semantics. The default constructor creates an instance with an empty cache path, `lru_only` policy, and the remaining values set to zero. */ //{@ PersistentCacheStats(); PersistentCacheStats(PersistentCacheStats const&); PersistentCacheStats(PersistentCacheStats&&) noexcept; PersistentCacheStats& operator=(PersistentCacheStats const&); PersistentCacheStats& operator=(PersistentCacheStats&&); //@} // Accessors instead of data members for ABI stability. /** @name Accessors */ //{@ /** \brief Returns the path to the cache directory. */ std::string cache_path() const; /** \brief Returns the discard policy (`lru_only` or `lru_ttl`). */ CacheDiscardPolicy policy() const noexcept; /** \brief Returns the number of entries (including expired ones). */ int64_t size() const noexcept; /** \brief Returns the size of all entries (including expired ones). */ int64_t size_in_bytes() const noexcept; /** \brief Returns the maximum size of the cache. */ int64_t max_size_in_bytes() const noexcept; /** \brief Returns the number of hits since the statistics were last reset. */ int64_t hits() const noexcept; /** \brief Returns the number of misses since the statistics were last reset. */ int64_t misses() const noexcept; /** \brief Returns the number of consecutive hits since the last miss. */ int64_t hits_since_last_miss() const noexcept; /** \brief Returns the number of consecutive misses since the last hit. */ int64_t misses_since_last_hit() const noexcept; /** \brief Returns the largest number of consecutive hits. */ int64_t longest_hit_run() const noexcept; /** \brief Returns the largest number of consecutive misses. */ int64_t longest_miss_run() const noexcept; /** \brief Returns the number of hit runs. */ int64_t hit_runs() const noexcept; /** \brief Returns the number of miss runs. */ int64_t miss_runs() const noexcept; /** \brief Returns a rolling average of the hit run length. */ double avg_hit_run_length() const noexcept; /** \brief Returns a rolling average of the miss run length. */ double avg_miss_run_length() const noexcept; /** \brief Returns the number of entries that were evicted due to being expired. */ int64_t ttl_evictions() const noexcept; /** \brief Returns the number of entries that were evicted due to being least recently used. */ int64_t lru_evictions() const noexcept; /** \brief Returns the timestamp of the most recent hit. */ std::chrono::system_clock::time_point most_recent_hit_time() const noexcept; /** \brief Returns the timestamp of the most recent miss. */ std::chrono::system_clock::time_point most_recent_miss_time() const noexcept; /** \brief Returns the time of the longest hit run. */ std::chrono::system_clock::time_point longest_hit_run_time() const noexcept; /** \brief Returns the time of the longest miss run. */ std::chrono::system_clock::time_point longest_miss_run_time() const noexcept; /** \brief Histogram of the size distribution of cache entries. The histogram uses a logarithmic scale and contains the number of entries in the cache, with entries grouped by size into a number of bins as follows: Index | Entry size in bytes ----- | ------------------- 0 | 1..9 1 | 10..19 2 | 20..29 ... | ... 9 | 90..99 10 | 100..199 11 | 200..200 ... | ... 18 | 900..999 19 | 1,000..1,999 20 | 2,000..2,999 ... | ... 27 | 9,000..9,999 28 | 10,000..19,999 29 | 20,000..29,999 ... | ... 72 | 900,000,000..999,999,999 73 | 1,000,000,000.. Index 0 contains the number of entries < 10 bytes. Thereafter, the histogram contains 9 bins for each power of 10, plus a final bin at index 73 that contains the number of entries ≥ 109 bytes. */ typedef std::vector Histogram; /** \brief Lower and upper bounds for the bins in the histogram. Each pair contains the lower and upper (inclusive) bound of the corresponding bin of the values returned by histogram(). */ typedef std::vector> HistogramBounds; /** \brief The number of bins in a histogram. */ static constexpr unsigned NUM_BINS = 74; /** \brief Returns a histogram for the entries in the cache. */ Histogram const& histogram() const noexcept; /** \brief Returns the bounds for each bin a histogram. This method returns the same vector each time; it is provided as a convenience method to make it easier to add labels to a histogram for display. The returned pairs use inclusive ranges, that is, `pair.second` is the largest possible size of the bin. */ static HistogramBounds const& histogram_bounds() noexcept; //@} private: PersistentCacheStats(std::shared_ptr const& p) noexcept; // We store a shared_ptr for efficiency. When the caller // retrieves the stats, we set p_ to point at the PersistentStringCacheStats // inside the cache. If the caller makes a copy or assigns, // we create a new instance, to provide value semantics. This means // that we don't have to copy all of the stats each time the caller // gets them. std::shared_ptr p_; bool internal_; // True if p_ points at the internal instance. // @cond friend class internal::PersistentStringCacheImpl; // For access to constructor // @endcond }; } // namespace core persistent-cache-cpp-1.0.5/include/core/persistent_string_cache.h000066400000000000000000000570471414021052300252000ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #pragma once #include #include #include #include namespace core { namespace internal { class PersistentStringCacheImpl; } // namespace internal /** \brief A cache of key-value pairs with persistent storage. PersistentStringCache provides a cache of `string` key-value pairs with a backing store. It is intended for caching arbitrary (possibly large) amounts of data, such as might be needed by a web browser cache. See @ref overview for a more detailed description. ### Usage example Typical use looks something like this: \code{.cpp} // Open cache or create it if it does not exist. auto c = core::PersistentStringCache::open("my_db", 1024 * 1024 * 1024, core::CacheDiscardPolicy::lru_only); // Look for an entry. If it doesn't exist, add it. string key = "some_key"; auto value = c->get(key); if (value) { cout << *value << endl; } else { string v = "some value"; c->put(key, v); } \endcode */ class PersistentStringCache { public: /** Convenience typedef for the return type of open(). */ typedef std::unique_ptr UPtr; /** \brief Simple pair of value and metadata. */ struct Data { /** \brief Stores the value of an entry. */ std::string value; /** \brief Stores the metadata of an entry. If no metadata exists for an entry, `metadata` is returned as the empty string when it is retrieved. */ std::string metadata; }; /** @name Copy and Assignment Cache instances are not copyable, but can be moved. \note The constructors are private. Use one of the open() static member functions to create or open a cache. */ //{@ PersistentStringCache(PersistentStringCache const&) = delete; PersistentStringCache& operator=(PersistentStringCache const&) = delete; PersistentStringCache(PersistentStringCache&&); PersistentStringCache& operator=(PersistentStringCache&&); //@} /** Destroys the instance. */ ~PersistentStringCache(); /** @name Creation Methods */ //{@ /** \brief Creates or opens a PersistentStringCache. If no cache exists on disk, it will be created; otherwise, the pre-existing cache contents are used. An existing cache can be opened only if `max_size_in_bytes` and `policy` have the same values they had when the cache was last closed. \param cache_path The path to a directory in which to store the cache. The contents of this directory are exlusively owned by the cache; do not create additional files or directories there. The directory need not exist when creating a new cache. \param max_size_in_bytes The maximum size in bytes for the cache. \param policy The discard policy for the cache (`lru_only` or `lru_ttl`). The discard policy cannot be changed once a cache has been created. The size of an entry is the sum of the sizes of its key, value, and metadata. The maximum size of the cache is the sum of the sizes of all its entries. \return A unique_ptr to the instance. \throws invalid_argument `max_size_in_bytes` is < 1. \throws logic_error `max_size_in_bytes` or `policy` do not match the settings of a pre-existing cache. */ static UPtr open(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); /** \brief Opens an existing PersistentStringCache. \param cache_path The path to a directory containing the existing cache. \return A unique_ptr to the instance. */ static UPtr open(std::string const& cache_path); //@} /** @name Accessors */ //{@ /** \brief Returns the value of an entry in the cache, provided the entry has not expired. \param key The key for the entry. \return A null value if the entry could not be retrieved; the value of the entry, otherwise. \throws invalid_argument `key` is the empty string. \note This operation updates the access time of the entry. */ Optional get(std::string const& key) const; /** \brief Returns the data for an entry in the cache, provided the entry has not expired. \param key The key for the entry. \return A null value if the entry could not be retrieved; the data of the entry, otherwise. If no metadata exists, `Data::metadata` is set to the empty string. \throws invalid_argument `key` is the empty string. \note This operation updates the access time of the entry. */ Optional get_data(std::string const& key) const; /** \brief Returns the metadata for an entry in the cache, provided the entry has not expired. \param key The key for the entry. \return A null value if the entry could not be retrieved; the metadata of the entry, otherwise. \throws invalid_argument `key` is the empty string. \note This operation does _not_ update the access time of the entry. \see touch() */ Optional get_metadata(std::string const& key) const; /** \brief Tests if an (unexpired) entry is in the cache. \param key The key for the entry. \return `true` if the entry is in the cache; `false` otherwise. \throws invalid_argument `key` is the empty string. \note This operation does _not_ update the access time of the entry. \see touch() */ bool contains_key(std::string const& key) const; /** \brief Returns the number of entries in the cache. \return The total number of entries in the cache. \note The returned count includes possibly expired entries. */ int64_t size() const noexcept; /** \brief Returns the number of bytes consumed by entries in the cache. \return The total number of bytes in the cache. \note The returned size includes possibly expired entries. */ int64_t size_in_bytes() const noexcept; /** \brief Returns the maximum size of the cache in bytes. \return The maximum number of bytes that can be stored in the cache. \see resize() */ int64_t max_size_in_bytes() const noexcept; /** \brief Returns an estimate of the disk space consumed by the cache. \return The approximate number of bytes used by the cache on disk. \note The returned size may be smaller than the eventual size if there are updates to the cache that have not yet been written to disk. */ int64_t disk_size_in_bytes() const; /** \brief Returns the discard policy of the cache. \return The discard policy (`lru_only` or `lru_ttl`). */ CacheDiscardPolicy discard_policy() const noexcept; /** \brief Returns statistics for the cache. The returned statistics are persistent and are restored the next time an existing cache is opened. Call clear_stats() to explicitly reset the statistics counters and time stamps to zero. \return An object that provides accessors to statistics and settings. \see clear_stats() */ PersistentCacheStats stats() const; //@} /** @name Modifiers */ //{@ /** \brief Adds or updates an entry. If an entry with the given key does not exist in the cache, it is added (possibly evicting a number of expired and/or older entries). If the entry still exists (whether expired or not), it is updated with the new value (and possibly expiry time). This operation deletes any metadata associated with the entry. \return `true` if the entry was added or updated. `false` if the policy is `lru_ttl` and `expiry_time` is in the past. \throws invalid_argument `key` is the empty string. \throws logic_error The size of the entry exceeds the maximum cache size. \throws logic_error The cache policy is `lru_only` and a non-infinite expiry time was provided. */ bool put(std::string const& key, std::string const& value, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Adds or updates an entry. \note This overload is provided to avoid the need to construct a string value. If an entry with the given key does not exist in the cache, it is added (possibly evicting a number of expired and/or older entries). If the entry still exists (whether expired or not), it is updated with the new value (and possibly expiry time). This operation deletes any metadata associated with the entry. \return `true` if the entry was added or updated. `false` if the policy is `lru_ttl` and `expiry_time` is in the past. \param key The key of the entry. \param value A pointer to the first byte of the value. \param size The size of the value in bytes. \param expiry_time The time at which the entry expires. \throws invalid_argument `key` is the empty string. \throws invalid_argument `value` is `nullptr`. \throws invalid_argument `size` is negative. \throws logic_error The size of the entry exceeds the maximum cache size. \throws logic_error The cache policy is `lru_only` and a non-infinite expiry time was provided. */ bool put(std::string const& key, char const* value, int64_t size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Adds or updates an entry and its metadata. If an entry with the given key does not exist in the cache, it is added (possibly evicting a number of expired and/or older entries). If the entry still exists (whether expired or not), it is updated with the new value and metadata (and possibly expiry time). \return `true` if the entry was added or updated. `false` if the policy is `lru_ttl` and `expiry_time` is in the past. \throws invalid_argument `key` is the empty string. \throws logic_error The sum of sizes of the entry and metadata exceeds the maximum cache size. \throws logic_error The cache policy is `lru_only` and a non-infinite expiry time was provided. */ bool put(std::string const& key, std::string const& value, std::string const& metadata, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Adds or updates an entry and its metadata. \note This overload is provided to avoid the need to construct strings for the value and metadata. If an entry with the given key does not exist in the cache, it is added (possibly evicting a number of expired and/or older entries). If the entry still exists (whether expired or not), it is updated with the new value and metadata (and possibly expiry time). \return `true` if the entry was added or updated. `false` if the policy is `lru_ttl` and `expiry_time` is in the past. \param key The key of the entry. \param value A pointer to the first byte of the value. \param value_size The size of the value in bytes. \param metadata A pointer to the first byte of the metadata. \param metadata_size The size of the metadata in bytes. \param expiry_time The time at which the entry expires. \throws invalid_argument `key` is the empty string. \throws logic_error The sum of sizes of the entry and metadata exceeds the maximum cache size. \throws logic_error The cache policy is `lru_only` and a non-infinite expiry time was provided. */ bool put(std::string const& key, char const* value, int64_t value_size, char const* metadata, int64_t metadata_size, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Function called by the cache to load an entry after a cache miss. */ typedef std::function Loader; /** \brief Atomically retrieves or stores a cache entry. `get_or_put` attempts to retrieve the value of a (non-expired) entry. If the entry can be found, it returns its value. Otherwise, it calls `load_func`, which is expected to add the entry to the cache. If the load function succeeds in adding the entry, the value added by the load function is returned. The load function is called by the application thread. \return A null value if the entry could not be retrieved or loaded; the value of the entry, otherwise. \throws runtime_error The load function threw an exception. \note The load function must (synchronously) call one of the overloaded `put` methods to add a new entry for the provided key. Calling any other method on the cache from within the load function causes undefined behavior. \warning This operation holds a lock on the cache while the load function runs. This means that, if multiple threads call into the cache, they will be blocked for the duration of the load function. */ Optional get_or_put(std::string const& key, Loader const& load_func); /** \brief Atomically retrieves or stores a cache entry. `get_or_put` attempts to retrieve the value and metadata of a (non-expired) entry. If the entry can be found, it returns its data. Otherwise, it calls `load_func`, which is expected to add the entry to the cache. If the load function succeeds in adding the entry, the data added by the load function is returned. The load function is called by the application thread. \return A null value if the entry could not be retrieved or loaded; the value and metadata of the entry, otherwise. \throws runtime_error The load function threw an exception. \note The load function must (synchronously) call one of the overloaded `put` methods to add a new entry for the provided key. Calling any other method on the cache from within the load function causes undefined behavior. \warning This operation holds a lock on the cache while the load function runs. This means that, if multiple threads call into the cache, they will be blocked for the duration of the load function. */ Optional get_or_put_data(std::string const& key, Loader const& load_func); /** \brief Adds or replaces the metadata for an entry. If a (non-expired) entry with the given key exists in the cache, its metadata is set to the provided value, replacing any previous metadata. \return `true` if the metadata was added or updated. `false` if the entry could not be found or was expired. \throws invalid_argument `key` is the empty string. \throws logic_error The new size of the entry would exceed the maximum cache size. \note This operation does _not_ update the access time of the entry. \see touch() */ bool put_metadata(std::string const& key, std::string const& metadata); /** \brief Adds or replaces the metadata for an entry. \note This overload is provided to avoid the need to construct a string for the metadata. If a (non-expired) entry with the given key exists in the cache, its metadata is set to the provided value, replacing any previous metadata. \param key The key of the entry. \param metadata A pointer to the first byte of the metadata. \param size The size of the metadata in bytes. \return `true` if the metadata was added or updated. `false` if the entry could not be found or was expired. \throws invalid_argument `key` is the empty string. \throws invalid_argument `metadata` is `nullptr`. \throws invalid_argument `size` is negative. \throws logic_error The new size of the entry would exceed the maximum cache size. \note This operation does _not_ update the access time of the entry. \see touch() */ bool put_metadata(std::string const& key, char const* metadata, int64_t size); /** \brief Removes an entry and returns its value. If a (non-expired) entry with the given key can be found, it is removed from the cache and its value returned. \return A null value if the entry could not be found; the value of the entry, otherwise. \throws invalid_argument `key` is the empty string. */ Optional take(std::string const& key); /** \brief Removes an entry and returns its value and metadata. If a (non-expired) entry with the given key can be found, it is removed from the cache and its data returned. If no metadata exists, `Data::metadata` is set to the empty string. \return A null value if the entry could not be retrieved; the value and metadata of the entry, otherwise. \throws invalid_argument `key` is the empty string. */ Optional take_data(std::string const& key); /** \brief Removes an entry and its associated metadata (if any). If a (non-expired) entry with the given key can be found, it is removed from the cache. \return `true` if the entry was removed; `false` if the entry could not be found or was expired. \throws invalid_argument `key` is the empty string. */ bool invalidate(std::string const& key); /** \brief Atomically removes the specified entries from the cache. \param keys A vector of keys for the entries to be removed. If the vector is empty, this operation is a no-op. If one or more keys are empty or specify non-existent entries, they are ignored. */ void invalidate(std::vector const& keys); /** \brief Atomically removes the specified entries from the cache. \param begin Iterator to the first key for the entries to be removed. \param end Iterator to the one-beyond-the-last key for the entries to be removed. If the iterator range is empty, this operation is a no-op. If one or more keys are empty or specify non-existent entries, they are ignored. */ template void invalidate(It begin, It end) { std::vector keys; while (begin < end) { keys.push_back(*begin++); } invalidate(keys); } /** \brief Atomically removes the specified entries from the cache. \param keys The keys for the entries to be removed. If `keys` is empty, this operation is a no-op. If one or more keys are empty or specify non-existent entries, they are ignored. */ void invalidate(std::initializer_list const& keys); /** \brief Deletes all entries from the cache. This operation completely empties the cache. \note Clearing the cache also resets the statistics counters. \see clear_stats() */ void invalidate(); /** \brief Updates the access time of an entry. If the entry specified by `key` is still in the cache (whether expired or not), it is marked as the most-recently used entry. If the policy is `lru_ttl`, the entry's expiry time is updated with the specified time (infinite expiry by default). \return `true` if the entry was updated; `false` if the entry could not be found or `expiry_time` is in the past. \throws invalid_argument `key` is the empty string. \throws logic_error `key` is the empty string. \throws logic_error The cache policy is `lru_only` and a non-infinite expiry time was provided. */ bool touch( std::string const& key, std::chrono::time_point expiry_time = std::chrono::system_clock::time_point()); /** \brief Resets all statistics counters. */ void clear_stats(); /** \brief Changes the maximum size of the cache. If `size_in_bytes` is greater or equal to max_size_in_bytes(), the cache size is set to `size_in_bytes`. If `size_in_bytes` is less than max_size_in_bytes(), the cache discards existing entries until the size falls to (or below) `size_in_bytes` and sets the cache size to the new value. \throws invalid_argument `size_in_bytes` is < 1 \note If the new size is less than the current size, this operation compacts the database to use the smallest possible amount of disk space. */ void resize(int64_t size_in_bytes); /** \brief Expires entries. Expires entries using the cache's expiration policy until the cache size falls to or below `used_size_in_bytes`. If `used_size_in_bytes` is less than the current cache size, this operation is a no-op. \throws invalid_argument `used_size_in_bytes` is < 0 \throws logic_error `used_size_in_bytes` is > max_size_in_bytes(). */ void trim_to(int64_t used_size_in_bytes); /** \brief Compacts the database. This operation compacts the database to consume as little disk space as possible. Note that this operation can be slow. (Compacting a 100 MB cache can take around ten seconds on a machine with a spinning-platter disk.) */ void compact(); //@} /** @name Monitoring cache activity The cache allows you to register one or more callback functions that are called when the cache contents change. \note Callback functions are called by the application thread that triggered the corresponding event. \warning Do not invoke operations on the cache from within a callback function. Doing so has undefined behavior. */ //{@ /** \brief The type of a handler function. \note Callback functions are called by the application thread that triggered the corresponding event. \warning Do not invoke operations on the cache from within a callback function. Doing so has undefined behavior. \param key The key of the entry. \param ev The event type. \param stats The cache statistics. Note that the `stats` parameter reflects the state of the cache _after_ the corresponding event. For example, for a `Put` event, `stats.size_in_bytes()` _includes_ the size of the added entry. */ typedef std::function EventCallback; /** \brief Installs a handler for one or more events. \param events A bitwise OR of the event types for which to install the handler. To install a handler for all events, you can use core::AllCacheEvents. \param cb The handler to install. To cancel an existing handler, pass `nullptr`. For example, to install a handler for `get` and `put` events, you could use: \code{.cpp} auto cache = PersistentStringCache::open("my_cache"); auto handler = [](string const& key, CacheEvent event, PersistentCacheStats const& stats) { // ... }; cache->set_handler(CacheEvent::get | CacheEvent::put, handler); \endcode \see CacheEvent */ void set_handler(CacheEvent events, EventCallback cb); //@} private: // @cond PersistentStringCache(std::string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy); PersistentStringCache(std::string const& cache_path); std::unique_ptr p_; // @endcond }; } // namespace core persistent-cache-cpp-1.0.5/src/000077500000000000000000000000001414021052300163155ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/src/CMakeLists.txt000066400000000000000000000000271414021052300210540ustar00rootroot00000000000000add_subdirectory(core) persistent-cache-cpp-1.0.5/src/core/000077500000000000000000000000001414021052300172455ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/src/core/CMakeLists.txt000066400000000000000000000006171414021052300220110ustar00rootroot00000000000000find_library(LEVELDB NAMES leveldb REQUIRED) add_subdirectory(internal) set(CACHE_SRC ${CACHE_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/persistent_cache_stats.cpp ${CMAKE_CURRENT_SOURCE_DIR}/persistent_string_cache.cpp ) add_library(${LIBNAME} STATIC ${CACHE_SRC}) target_link_libraries(${LIBNAME} ${LEVELDB}) install(TARGETS ${LIBNAME} DESTINATION lib/${CMAKE_LIBRARY_ARCHITECTURE}) persistent-cache-cpp-1.0.5/src/core/internal/000077500000000000000000000000001414021052300210615ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/src/core/internal/CMakeLists.txt000066400000000000000000000002321414021052300236160ustar00rootroot00000000000000set(CACHE_INTERNAL_SRC ${CMAKE_CURRENT_SOURCE_DIR}/persistent_string_cache_impl.cpp ) set(CACHE_SRC ${CACHE_SRC} ${CACHE_INTERNAL_SRC} PARENT_SCOPE) persistent-cache-cpp-1.0.5/src/core/internal/persistent_string_cache_impl.cpp000066400000000000000000001512511414021052300275240ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #include #include #include #include #include /* We have three tables and two secondary indexes in the DB: - Key -> Value The Values table maps keys to values. - Key -> The Data table maps keys to the access time, expire time, and entry size. (Size is the sum of key, value, and metadata sizes.) - Key -> Metadata The Metadata table maps keys to metadata for the entry. If no metadata exists for an entry, there is no row in the table. - -> Size The Atime index provides access in order of oldest-to-newest access time. This allows efficient trimming based on LRU order. - -> Size The Etime index provides access in order of soonest-to-latest expiry time. This allows efficient trimming of expired entries. For lru_only, no entry is added to this index, and the corresponding expiry time in the Data table is 0. For lru_ttl, only entries that actually do have an expiry time are added. The tables and indexes each map to a different region of the leveldb based on a prefix. Tuple entries are separated by spaces. Entries are sorted in lexicographical order by the DB; to ensure correct numerical comparison for the secondary indexes, times are in milliseconds since the epoch, with fixed-width width zero padding and 13 decimal digits. (That works out to more than 316 years past the epoch.) Some examples to illustrate how it hangs together with lru_ttl. (Note that, in reality, all four tables really sit inside the single leveldb table, separated by the prefixes of their keys. They are shown as separate tables below to make things easier to read.) At time 0010, insert Bjarne -> Stroustrup, expires 1010, at time 0020, insert Andy -> Koenig, expires 2020, at time 0030, insert Scott -> Meyers, does not expire at time 0040, insert Stan -> Lippman, expires 1040 Values: Data: Key | Value Key | Access time | Expiry time | Size --------+----------- --------+--------------------------------- AAndy | Koenig BAndy | 20 | 2020 | 10 ABjarne | Stroustrup BBjarne | 10 | 1010 | 16 AScott | Meyers BScott | 30 | 0 | 11 AStan | Lippman BStan | 40 | 1040 | 11 Atime index: Etime index: Key | Size Key | Size ----------------------+----- ----------------------+----- D0000000000010 Bjarne | 16 E0000000001010 Bjarne | 16 D0000000000020 Andy | 10 E0000000001040 Stan | 11 D0000000000030 Scott | 11 E0000000002020 Andy | 10 D0000000000040 Stan | 11 Note that, because the expiry time for Scott is infinite, no entry appears in the Etime index. At time 100, we call get("Bjarne"). This updates the Data table and Atime index with the new access time. (The Values table and Etime index are unchanged.) Values: Data: Key | Value Key | Access time | Expiry time | Size --------+----------- --------+--------------------------------- AAndy | Koenig BAndy | 20 | 2020 | 10 ABjarne | Stroustrup BBjarne | 100 | 1010 | 16 AScott | Meyers BScott | 30 | 0 | 11 AStan | Lippman BStan | 40 | 1040 | 11 Atime index: Etime index: Key | Size Key | Size ----------------------+----- ----------------------+----- D0000000000020 Andy | 10 E0000000001010 Bjarne | 16 D0000000000030 Scott | 11 E0000000001040 Stan | 11 D0000000000040 Stan | 11 E0000000002020 Andy | 10 D0000000000100 Bjarne | 16 In other words, the Atime and Etime indexes are always sorted in earliest-to-latest order of expiry; this allows us to efficiently trim the cache once it is full. The sizes are stored redundantly so we can efficiently determine the point at which we have removed enough entries in order to make room for a new one. If this example were to use lru_only, the Etime index would remain empty, and the expiry times in the Data table would all be chrono::duration_cast(chrono::time_point()).count(). That value typically is zero (but this is not guaranteed by the standard). The Metadata table simply maps each key to its metadata. If no metadata exists for an entry, there is no corresponding row in the table. For example, if Andy and Scott had metadata, but Stan and Bjarne didn't, we'd have: Metadata table: Key | Metadata --------+---------- CAndy | CScott | */ using namespace std; namespace core { namespace internal { namespace { static string const class_name = "PersistentStringCache"; // For exception messages static leveldb::WriteOptions write_options; static leveldb::ReadOptions read_options; // Schema version. If the way things are written to leveldb changes, the // schema version here must be changed, too. If an existing cache is opened // with a different schema version, the cache is simply thrown away, so // it will automatically be re-created using the latest schema. static int const SCHEMA_VERSION = 3; // Increment whenever schema changes! // Prefixes to divide the key space into logical tables/indexes. // All prefixes must have length 1. The end prefix must be // the "one past the end" value for the prefix range. For example, // "B" is the first prefix that can't be an entry in the Values table. // // Do not change the prefix without also checking that ALL_BEGIN and // ALL_END are still correct! static string const VALUES_BEGIN = "A"; static string const VALUES_END = "B"; static string const DATA_BEGIN = "B"; static string const DATA_END = "C"; static string const METADATA_BEGIN = "C"; static string const METADATA_END = "D"; static string const ATIME_BEGIN = "D"; static string const ATIME_END = "E"; static string const ETIME_BEGIN = "E"; static string const ETIME_END = "F"; // We store the stats so they are not lost across process re-starts. static string const STATS_BEGIN = "X"; static string const STATS_END = "Y"; // The settings range stores data about the cache itself, such as // max size and expiration policy. The prefix for this // range must be outside the range [ALL_BEGIN..ALL_END). // The schema version is there so we can change the way things are written into leveldb // and detect when an old cache is opened with a newer version. static string const SETTINGS_BEGIN = "Y"; static string const SETTINGS_END = "Z"; // This key stores a dirty flag that we set after successful open // and clear in the destructor. This allows to detect, on start-up // if we shut down cleanly. static string const DIRTY_FLAG = "!DIRTY"; // These span the entire range of keys in all tables and stats (except settings and dirty flag). static string const ALL_BEGIN = VALUES_BEGIN; // Must be lowest prefix for all tables and indexes, incl stats. static string const ALL_END = SETTINGS_BEGIN; // Must be highest prefix for all tables and indexes, incl stats. static string const SETTINGS_MAX_SIZE = SETTINGS_BEGIN + "MAX_SIZE"; static string const SETTINGS_POLICY = SETTINGS_BEGIN + "POLICY"; static string const SETTINGS_SCHEMA_VERSION = SETTINGS_BEGIN + "SCHEMA_VERSION"; static string const STATS_VALUES = STATS_BEGIN + "VALUES"; // Simple struct to serialize/deserialize a time-key tuple. // For the stringified representation, time and key are // separated by a space. struct TimeKeyTuple { int64_t time; // msec since the epoch string key; TimeKeyTuple(int64_t t, string const& s) : time(t) , key(s) { } TimeKeyTuple(string const& s) { auto pos = s.find(' '); assert(pos != string::npos); string t(s.substr(0, pos)); istringstream is(t); is >> time; assert(!is.bad()); key = s.substr(pos + 1); } TimeKeyTuple(TimeKeyTuple const&) = default; TimeKeyTuple(TimeKeyTuple&&) = default; TimeKeyTuple& operator=(TimeKeyTuple const&) = default; TimeKeyTuple& operator=(TimeKeyTuple&&) = default; string to_string() const { // We zero-fill the time, so entries collate lexicographically in old-to-new order. ostringstream os; os << setfill('0') << setw(13) << time << " " << key; return os.str(); } }; // Key creation methods. These methods return the key into the corresponding // table or index with the correct prefix and with tuple keys concatenated // with a space separator. string k_data(string const& key) { return DATA_BEGIN + key; } string k_metadata(string const& key) { return METADATA_BEGIN + key; } string k_atime_index(int64_t atime, string const& key) { return ATIME_BEGIN + TimeKeyTuple(atime, key).to_string(); } string k_etime_index(int64_t etime, string const& key) { return ETIME_BEGIN + TimeKeyTuple(etime, key).to_string(); } // Little helpers to get milliseconds since the epoch. int64_t ticks(chrono::time_point tp) noexcept { return chrono::duration_cast(tp.time_since_epoch()).count(); } int64_t now_ticks() noexcept { return ticks(chrono::system_clock::now()); } // Usually zero, but the standard doesn't guarantee this. static auto const clock_origin = ticks(chrono::system_clock::time_point()); int64_t epoch_ticks() noexcept { return clock_origin; } typedef std::unique_ptr IteratorUPtr; #ifndef NDEBUG // For assertions, so we can verify that num_entries_ matches the sum of entries in the histogram. int64_t hist_sum(PersistentCacheStats::Histogram const& h) noexcept { int64_t size = 0; for (auto num : h) { size += num; } return size; } #endif } // namespace void PersistentStringCacheImpl::init_stats() { int64_t num = 0; int64_t size = 0; // If we shut down cleanly last time, read the saved stats values. bool is_dirty = read_dirty_flag(); if (!is_dirty) { read_stats(); } else { // We didn't shut down cleanly or the cache is new. // Run over the Atime index (it's smaller than the Data table) // and count the number of entries and bytes, and initialize // the histogram. IteratorUPtr it(db_->NewIterator(read_options)); leveldb::Slice const atime_prefix(ATIME_BEGIN); it->Seek(atime_prefix); while (it->Valid() && it->key().starts_with(atime_prefix)) { ++num; auto bytes = stoll(it->value().ToString()); size += bytes; stats_->hist_increment(bytes); it->Next(); } throw_if_error(it->status(), "cannot initialize cache"); stats_->num_entries_ = num; stats_->cache_size_ = size; } assert(stats_->num_entries_ == hist_sum(stats_->hist_)); } // Open existing database or create an empty one. PersistentStringCacheImpl::PersistentStringCacheImpl(string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy, PersistentStringCache* pimpl) : pimpl_(pimpl) , stats_(make_shared()) { stats_->cache_path_ = cache_path; if (max_size_in_bytes < 1) { throw_invalid_argument("invalid max_size_in_bytes (" + to_string(max_size_in_bytes) + "): value must be > 0"); } stats_->max_cache_size_ = max_size_in_bytes; stats_->policy_ = policy; leveldb::Options options; options.create_if_missing = true; // For small caches, reduce memory consumption by reducing the size of the internal block cache. // The block cache size is at least 512 kB. For caches 5-80 MB, it is 10% of the nominal cache size. // For caches > 80 MB, the block cache is left at the default of 8 MB. size_t block_cache_size = max_size_in_bytes / 10; if (block_cache_size < 512 * 1024) { block_cache_size = 512 * 1024; } if (block_cache_size < 8 * 1024 * 1024) { block_cache_.reset(leveldb::NewLRUCache(block_cache_size)); options.block_cache = block_cache_.get(); } init_db(options); if (cache_is_new()) { write_version(); write_settings(); write_stats(); } else { check_version(); // Wipes DB if version doesn't match. read_settings(); // For an already-existing cache, check that size and policy match. if (stats_->max_cache_size_ != max_size_in_bytes) { throw_logic_error(string("existing cache opened with different max_size_in_bytes (") + to_string(max_size_in_bytes) + "), existing size = " + to_string(stats_->max_cache_size_)); } if (stats_->policy_ != policy) { auto to_string = [](CacheDiscardPolicy p) { return p == core::CacheDiscardPolicy::lru_only ? "lru_only" : "lru_ttl"; }; string msg = string("existing cache opened with different policy (") + to_string(policy) + "), existing policy = " + to_string(stats_->policy_); throw_logic_error(msg); } } init_stats(); write_dirty_flag(true); } // Open existing database. PersistentStringCacheImpl::PersistentStringCacheImpl(string const& cache_path, PersistentStringCache* pimpl) : pimpl_(pimpl) , stats_(make_shared()) { stats_->cache_path_ = cache_path; init_db(leveldb::Options()); // Throws if DB doesn't exist. check_version(); // Wipes DB if version doesn't match. read_settings(); init_stats(); write_dirty_flag(true); } PersistentStringCacheImpl::~PersistentStringCacheImpl() { try { write_stats(); write_dirty_flag(false); } // LCOV_EXCL_START catch (std::exception const& e) { cerr << make_message(string("~PersistentStringCacheImpl(): ") + e.what()) << endl; } catch (...) { cerr << make_message("~PersistentStringCacheImpl(): unknown exception") << endl; } // LCOV_EXCL_STOP } bool PersistentStringCacheImpl::get(string const& key, string& value) const { return get(key, value, nullptr); } bool PersistentStringCacheImpl::get(string const& key, string& value, string* metadata) const { if (key.empty()) { throw_invalid_argument("get(): key must be non-empty"); } lock_guard lock(mutex_); string data_key = k_data(key); DataTuple dt; bool found = get_value_and_metadata(key, dt, value, metadata); if (!found) { stats_->inc_misses(); call_handler(key, CacheEventIndex::miss); return false; } // Don't return expired entry. int64_t new_atime = now_ticks(); if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks() && dt.etime <= new_atime) { call_handler(key, CacheEventIndex::miss); stats_->inc_misses(); return false; } leveldb::WriteBatch batch; batch.Delete(k_atime_index(dt.atime, key)); // Delete old atime entry dt.atime = new_atime; dt.size = key.size() + value.size(); if (metadata) { dt.size += metadata->size(); } batch.Put(data_key, dt.to_string()); batch.Put(k_atime_index(dt.atime, key), to_string(dt.size)); auto s = db_->Write(write_options, &batch); throw_if_error(s, "put()"); stats_->inc_hits(); call_handler(key, CacheEventIndex::get); return true; } bool PersistentStringCacheImpl::get_metadata(string const& key, string& metadata) const { if (key.empty()) { throw_invalid_argument("get_metadata(): key must be non-empty"); } lock_guard lock(mutex_); string data_key = k_data(key); bool found; auto dt = get_data(data_key, found); if (!found) { return false; } // Don't return expired entry. if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks() && dt.etime <= now_ticks()) { return false; } auto s = db_->Get(read_options, k_metadata(key), &metadata); throw_if_error(s, "get_metadata()"); return !s.IsNotFound(); } bool PersistentStringCacheImpl::contains_key(string const& key) const { if (key.empty()) { throw_invalid_argument("contains_key(): key must be non-empty"); } lock_guard lock(mutex_); string data_key = k_data(key); bool found; auto dt = get_data(data_key, found); if (!found) { return false; } if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks() && dt.etime <= now_ticks()) { return false; // Expired entries are not returned. } return true; } int64_t PersistentStringCacheImpl::size() const noexcept { lock_guard lock(mutex_); return stats_->num_entries_; } int64_t PersistentStringCacheImpl::size_in_bytes() const noexcept { lock_guard lock(mutex_); return stats_->cache_size_; } int64_t PersistentStringCacheImpl::max_size_in_bytes() const noexcept { lock_guard lock(mutex_); return stats_->max_cache_size_; } int64_t PersistentStringCacheImpl::disk_size_in_bytes() const { lock_guard lock(mutex_); leveldb::Range everything(ALL_BEGIN, SETTINGS_END); array sizes = {{0}}; db_->GetApproximateSizes(&everything, 1, sizes.data()); return sizes[0]; } CacheDiscardPolicy PersistentStringCacheImpl::discard_policy() const noexcept { return stats_->policy_; // Immutable } PersistentCacheStats PersistentStringCacheImpl::stats() const { lock_guard lock(mutex_); // We make a copy here so values can't change underneath the caller. return PersistentCacheStats(make_shared(*stats_)); } bool PersistentStringCacheImpl::put(string const& key, string const& value, chrono::time_point expiry_time) { return put(key, value.data(), value.size(), nullptr, 0, expiry_time); } bool PersistentStringCacheImpl::put(string const& key, char const* value_data, int64_t value_size, chrono::time_point expiry_time) { return put(key, value_data, value_size, nullptr, 0, expiry_time); } bool PersistentStringCacheImpl::put(string const& key, string const& value, string const* metadata, chrono::time_point expiry_time) { assert(metadata); return put(key, value.data(), value.size(), metadata->data(), metadata->size(), expiry_time); } bool PersistentStringCacheImpl::put(string const& key, char const* value_data, int64_t value_size, char const* metadata_data, int64_t metadata_size, chrono::time_point expiry_time) { if (key.empty()) { throw_invalid_argument("put(): key must be non-empty"); } if (!value_data) { throw_invalid_argument("put(): value must not be nullptr"); } if (value_size < 0) { throw_invalid_argument("put(): invalid negative value size: " + to_string(value_size)); } if (metadata_data && metadata_size < 0) { throw_invalid_argument("put(): invalid negative metadata size: " + to_string(metadata_size)); } int64_t new_size = key.size() + value_size; if (metadata_data) { new_size += metadata_size; } if (new_size > stats_->max_cache_size_) { throw_logic_error(string("put(): cannot add ") + to_string(new_size) + "-byte record to cache with maximum size of " + to_string(stats_->max_cache_size_)); } auto etime = ticks(expiry_time); if (stats_->policy_ == CacheDiscardPolicy::lru_only && etime != epoch_ticks()) { throw_logic_error(string("put(): policy is lru_only, but expiry_time (") + to_string(etime) + ") is not infinite"); } lock_guard lock(mutex_); auto atime = now_ticks(); if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && etime != epoch_ticks() && etime <= atime) { return false; // Already expired, so don't add it. } // The entry may or may not exist already. // Work out how many bytes of space we need. int64_t bytes_needed = new_size; string prefixed_key = k_data(key); bool found; auto old_data = get_data(prefixed_key, found); if (found) { bytes_needed = max(new_size - old_data.size, int64_t(0)); // new_size could be < old size } auto avail_bytes = stats_->max_cache_size_ - stats_->cache_size_; // Make room to add or replace the entry. if (bytes_needed > avail_bytes) { delete_at_least(bytes_needed - avail_bytes, key); // Don't delete the entry about to be updated! } leveldb::WriteBatch batch; // Update the Data table. DataTuple new_meta(atime, etime, new_size); batch.Put(prefixed_key, new_meta.to_string()); // Add or replace the entry in the Values table. prefixed_key[0] = VALUES_BEGIN[0]; // Avoid string copy. batch.Put(prefixed_key, leveldb::Slice(value_data, value_size)); // Update metadata. prefixed_key[0] = METADATA_BEGIN[0]; // Avoid string copy. batch.Delete(prefixed_key); // In case there was metadata previously. if (metadata_data) { batch.Put(prefixed_key, leveldb::Slice(metadata_data, metadata_size)); } // Update the Atime index. string atime_key = k_atime_index(atime, key); if (found) { batch.Delete(k_atime_index(old_data.atime, key)); } batch.Put(atime_key, to_string(new_size)); // Update the Etime index. if (stats_->policy_ == CacheDiscardPolicy::lru_ttl) { if (found && old_data.etime != epoch_ticks()) { batch.Delete(k_etime_index(old_data.etime, key)); } // Etime index is not written to for non-expiring entries. if (etime != epoch_ticks()) { batch.Put(k_etime_index(etime, key), to_string(new_size)); } } // Write the batch. auto s = db_->Write(write_options, &batch); throw_if_error(s, "put()"); // Update cache size and number of entries; stats_->cache_size_ = stats_->cache_size_ - old_data.size + new_size; stats_->hist_increment(new_size); if (!found) { ++stats_->num_entries_; } else { stats_->hist_decrement(old_data.size); } assert(stats_->num_entries_ >= 0); assert(stats_->num_entries_ == hist_sum(stats_->hist_)); assert(stats_->cache_size_ >= 0); assert(stats_->cache_size_ <= stats_->max_cache_size_); assert(stats_->cache_size_ == 0 || stats_->num_entries_ != 0); assert(stats_->num_entries_ == 0 || stats_->cache_size_ != 0); call_handler(key, CacheEventIndex::put); return true; } bool PersistentStringCacheImpl::get_or_put(string const& key, string& value, PersistentStringCache::Loader load_func) { return get_or_put(key, value, nullptr, load_func); } bool PersistentStringCacheImpl::get_or_put(string const& key, string& value, string* metadata, PersistentStringCache::Loader load_func) { if (key.empty()) { throw_invalid_argument("get_or_put(): key must be non-empty"); } lock_guard lock(mutex_); // Call the normal get() here, so the hit/miss counters and callbacks are correct. if (get(key, value, metadata)) { return true; } try { load_func(key, *pimpl_); // Expected to put the value. } catch (std::exception const& e) { throw runtime_error(make_message(string("get_or_put(): load_func exception: ") + e.what())); } catch (...) { throw runtime_error(make_message("get_or_put(): load_func: unknown exception")); } // We go for the raw DB here, to avoid counting an extra hit or miss. DataTuple dt; bool loaded = get_value_and_metadata(key, dt, value, metadata); return loaded; } bool PersistentStringCacheImpl::put_metadata(std::string const& key, std::string const& metadata) { return put_metadata(key, metadata.data(), metadata.size()); } bool PersistentStringCacheImpl::put_metadata(std::string const& key, const char* metadata, int64_t metadata_size) { if (key.empty()) { throw_invalid_argument("put_metadata(): key must be non-empty"); } if (!metadata) { throw_invalid_argument("put_metadata(): metadata must not be nullptr"); } if (metadata_size < 0) { throw_invalid_argument("put_metadata(): invalid negative size: " + to_string(metadata_size)); } lock_guard lock(mutex_); string data_key = k_data(key); bool found; auto dt = get_data(data_key, found); if (!found) { return false; } int64_t old_meta_size = 0; IteratorUPtr it(db_->NewIterator(read_options)); string metadata_key = k_metadata(key); it->Seek(metadata_key); if (it->Valid() && it->key().ToString() == metadata_key) { old_meta_size = it->value().size(); } int64_t new_meta_size = metadata_size; if (dt.size - old_meta_size + new_meta_size > stats_->max_cache_size_) { throw_logic_error(string("put_metadata(): cannot add ") + to_string(new_meta_size) + "-byte metadata: record size (" + to_string(dt.size - old_meta_size + new_meta_size) + ") exceeds maximum cache size of " + to_string(stats_->max_cache_size_)); } int64_t original_size = dt.size; dt.size = dt.size - old_meta_size + new_meta_size; if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks() && dt.etime <= now_ticks()) { return false; // Entry has expired. } // The new entry may be larger than the old one. If so, we may need to // evict some other entries to make room. However, we exclude this // record from trimming so we don't end up trimming the entry // that's about to be modified. if (new_meta_size > old_meta_size) { auto avail_bytes = stats_->max_cache_size_ - stats_->cache_size_; int64_t bytes_needed = new_meta_size - old_meta_size; if (bytes_needed > avail_bytes) { bytes_needed = min(bytes_needed, avail_bytes); delete_at_least(bytes_needed, key); // Don't delete the entry about to be updated! } } leveldb::WriteBatch batch; if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks()) { it->Seek(k_etime_index(dt.etime, key)); assert(it->Valid()); assert(it->key().ToString() == k_etime_index(dt.etime, key)); batch.Put(it->key(), to_string(dt.size)); // Update Etime index with new size (expiry time is not modified). } batch.Put(data_key, dt.to_string()); // Update data. batch.Put(metadata_key, leveldb::Slice(metadata, metadata_size)); // Update metadata. it->Seek(k_atime_index(dt.atime, key)); assert(it->Valid()); assert(it->key().ToString() == k_atime_index(dt.atime, key)); batch.Put(it->key(), to_string(dt.size)); // Update Atime index with new size (access time is not modified). auto s = db_->Write(write_options, &batch); throw_if_error(s, "put_metadata(): batch write error"); stats_->cache_size_ = stats_->cache_size_ - old_meta_size + new_meta_size; stats_->hist_increment(dt.size); stats_->hist_decrement(original_size); assert(stats_->num_entries_ >= 0); assert(stats_->num_entries_ == hist_sum(stats_->hist_)); assert(stats_->cache_size_ >= 0); assert(stats_->cache_size_ <= stats_->max_cache_size_); assert(stats_->cache_size_ == 0 || stats_->num_entries_ != 0); assert(stats_->num_entries_ == 0 || stats_->cache_size_ != 0); return true; } bool PersistentStringCacheImpl::take(string const& key, string& value) { return take(key, value, nullptr); } bool PersistentStringCacheImpl::take(string const& key, string& value, string* metadata) { if (key.empty()) { throw_invalid_argument("take(): key must be non-empty"); } lock_guard lock(mutex_); string data_key = k_data(key); DataTuple dt; string val; bool found = get_value_and_metadata(key, dt, val, metadata); if (!found) { stats_->inc_misses(); call_handler(key, CacheEventIndex::miss); return false; } // Delete the entry whether it expired or not. Seeing that we have just done // a lot of work finding it, we may as well finish the job. delete_entry(key, dt); if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks() && dt.etime <= now_ticks()) { stats_->inc_misses(); call_handler(key, CacheEventIndex::invalidate); call_handler(key, CacheEventIndex::miss); return false; // Expired entries are hidden. } stats_->inc_hits(); value = move(val); call_handler(key, CacheEventIndex::get); call_handler(key, CacheEventIndex::invalidate); assert(stats_->num_entries_ == hist_sum(stats_->hist_)); return true; } bool PersistentStringCacheImpl::invalidate(string const& key) { if (key.empty()) { throw_invalid_argument("invalidate(): key must be non-empty"); } lock_guard lock(mutex_); string prefixed_key = k_data(key); bool found; auto dt = get_data(prefixed_key, found); if (!found) { return false; } // Delete the entry whether it expired or not. Seeing that we have just done // a lot of work finding it, we may as well finish the job. delete_entry(key, dt); call_handler(key, CacheEventIndex::invalidate); if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && dt.etime != epoch_ticks() && dt.etime < now_ticks()) { return false; // Expired entries are hidden. } assert(stats_->num_entries_ == hist_sum(stats_->hist_)); return true; } void PersistentStringCacheImpl::invalidate(vector const& keys) { invalidate(keys.begin(), keys.end()); } template void PersistentStringCacheImpl::invalidate(It begin, It end) { lock_guard lock(mutex_); leveldb::WriteBatch batch; for (auto&& it = begin; it < end; ++it) { if (it->empty()) { continue; } bool found; auto dt = get_data(k_data(*it), found); if (!found) { continue; } batch_delete(*it, dt, batch); // Update cache size and entries. stats_->hist_decrement(dt.size); stats_->cache_size_ -= dt.size; assert(stats_->cache_size_ >= 0); assert(stats_->cache_size_ <= stats_->max_cache_size_); --stats_->num_entries_; assert(stats_->num_entries_ >= 0); assert(stats_->num_entries_ == hist_sum(stats_->hist_)); assert(stats_->cache_size_ == 0 || stats_->num_entries_ != 0); assert(stats_->num_entries_ == 0 || stats_->cache_size_ != 0); call_handler(*it, CacheEventIndex::invalidate); } auto s = db_->Write(write_options, &batch); throw_if_error(s, "invalidate(): batch write error"); } void PersistentStringCacheImpl::invalidate(initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } void PersistentStringCacheImpl::invalidate() { lock_guard lock(mutex_); { int64_t count = 0; leveldb::WriteBatch batch; int64_t const batch_size = 1000; PersistentStringCache::EventCallback cb = handlers_[static_cast::type>(CacheEventIndex::invalidate)]; IteratorUPtr it(db_->NewIterator(read_options)); it->Seek(ALL_BEGIN); leveldb::Slice const atime_prefix = ATIME_BEGIN; leveldb::Slice const all_end = ALL_END; while (it->Valid() && it->key().compare(all_end) < 0) { auto key = it->key(); batch.Delete(key); if (cb && key.starts_with(atime_prefix)) { TimeKeyTuple atk(key.ToString().substr(1)); --stats_->num_entries_; auto size = stoll(it->value().ToString()); stats_->cache_size_ -= size; call_handler(atk.key, CacheEventIndex::invalidate); } if (++count == batch_size) { auto s = db_->Write(write_options, &batch); throw_if_error(s, "invalidate(): batch write error"); batch.Clear(); count = 0; } it->Next(); } throw_if_error(it->status(), "invalidate(): iterator error"); if (count != 0) { auto s = db_->Write(write_options, &batch); throw_if_error(s, "invalidate(): final batch write error"); } } // Close batch stats_->num_entries_ = 0; stats_->hist_clear(); stats_->cache_size_ = 0; // Clear ephemeral stats too. stats_->clear(); write_stats(); } bool PersistentStringCacheImpl::touch(string const& key, chrono::time_point expiry_time) { if (key.empty()) { throw_invalid_argument("touch(): key must be non-empty"); } int64_t new_etime = ticks(expiry_time); if (stats_->policy_ == CacheDiscardPolicy::lru_only && new_etime != epoch_ticks()) { throw_logic_error(string("touch(): policy is lru_only, but expiry_time (") + to_string(new_etime) + ") is not infinite"); } lock_guard lock(mutex_); string data_key = k_data(key); bool found; auto dt = get_data(data_key, found); if (!found) { return false; } int64_t now = now_ticks(); if (stats_->policy_ == CacheDiscardPolicy::lru_ttl && new_etime != epoch_ticks() && new_etime <= now) { return false; // New expiry time is already older than the time now. } leveldb::WriteBatch batch; string size = to_string(dt.size); batch.Delete(k_atime_index(dt.atime, key)); // Delete old Atime index entry. batch.Put(k_atime_index(now, key), size); // Write new Atime index entry. if (stats_->policy_ == CacheDiscardPolicy::lru_ttl) { batch.Delete(k_etime_index(dt.etime, key)); // Delete old Etime index entry. if (new_etime != epoch_ticks()) { batch.Put(k_etime_index(new_etime, key), size); // Write new Etime index entry. } } dt.atime = now; dt.etime = new_etime; batch.Put(data_key, dt.to_string()); // Write new data. auto s = db_->Write(write_options, &batch); throw_if_error(s, "touch(): batch write error"); call_handler(key, CacheEventIndex::touch); return true; } void PersistentStringCacheImpl::clear_stats() noexcept { lock_guard lock(mutex_); stats_->clear(); write_stats(); } void PersistentStringCacheImpl::resize(int64_t size_in_bytes) { if (size_in_bytes < 1) { throw_invalid_argument("resize(): invalid size_in_bytes (" + to_string(size_in_bytes) + "): value must be > 0"); } lock_guard lock(mutex_); if (size_in_bytes < stats_->max_cache_size_) { trim_to(size_in_bytes); db_->CompactRange(nullptr, nullptr); // Avoid bulk deletions slowing down subsequent accesses. } auto s = db_->Put(write_options, SETTINGS_MAX_SIZE, to_string(size_in_bytes)); throw_if_error(s, "resize(): cannot write max size"); stats_->max_cache_size_ = size_in_bytes; assert(stats_->num_entries_ == hist_sum(stats_->hist_)); } void PersistentStringCacheImpl::trim_to(int64_t used_size_in_bytes) { if (used_size_in_bytes < 0) { throw_invalid_argument("trim_to(): invalid used_size_in_bytes (" + to_string(used_size_in_bytes) + "): value must be >= 0"); } if (used_size_in_bytes > stats_->max_cache_size_) { throw_logic_error(string("trim_to(): invalid used_size_in_bytes (") + to_string(used_size_in_bytes) + "): value must be <= max_size_in_bytes (" + to_string(stats_->max_cache_size_) + ")"); } lock_guard lock(mutex_); if (used_size_in_bytes < stats_->cache_size_) { delete_at_least(stats_->cache_size_ - used_size_in_bytes); } assert(stats_->num_entries_ == hist_sum(stats_->hist_)); } void PersistentStringCacheImpl::compact() { lock_guard lock(mutex_); db_->CompactRange(nullptr, nullptr); } void PersistentStringCacheImpl::set_handler(CacheEvent events, PersistentStringCache::EventCallback cb) { static constexpr auto limit = underlying_type::type(CacheEvent::END_); auto evs = underlying_type::type(events); if (evs == 0 || evs > limit - 1) { throw_invalid_argument("set_handler(): invalid events (" + to_string(evs) + "): value must be in the range [1.." + to_string(limit - 1) + "]"); } lock_guard lock(mutex_); static constexpr auto index_limit = underlying_type::type(CacheEventIndex::END_); for (underlying_type::type i = 0; i < index_limit; ++i) { if ((evs >> i) & 1) { handlers_[i] = cb; } } } void PersistentStringCacheImpl::init_db(leveldb::Options options) { #ifndef NDEBUG options.paranoid_checks = true; read_options.verify_checksums = true; #endif leveldb::DB* db; auto s = leveldb::DB::Open(options, stats_->cache_path_, &db); throw_if_error(s, "cannot open or create cache"); db_.reset(db); } bool PersistentStringCacheImpl::cache_is_new() const { string val; auto s = db_->Get(read_options, SETTINGS_SCHEMA_VERSION, &val); throw_if_error(s, "cannot read schema version"); return s.IsNotFound(); } void PersistentStringCacheImpl::write_version() { auto s = db_->Put(write_options, SETTINGS_SCHEMA_VERSION, to_string(SCHEMA_VERSION)); throw_if_error(s, "cannot read schema version"); } // Check if the version of the DB matches the expected version. // Pre: Version exists in the DB. // If the version can be read and make sense as a number, but // differs from the expected version, wipe the data (but // not the settings). // If the version can be read, but doesn't parse as a number, throw. // If the version matches the expected version, do nothing. void PersistentStringCacheImpl::check_version() { // Check schema version. string val = "not found"; auto s = db_->Get(read_options, SETTINGS_SCHEMA_VERSION, &val); throw_if_error(s, "cannot read schema version"); assert(!s.IsNotFound()); int old_version; try { old_version = stoi(val); } catch (std::exception const&) { throw_corrupt_error("check_version(): bad version: \"" + val + "\""); } if (old_version != SCHEMA_VERSION) { // Wipe all tables and stats (but not settings). leveldb::WriteBatch batch; IteratorUPtr it(db_->NewIterator(read_options)); it->Seek(ALL_BEGIN); leveldb::Slice const all_end(ALL_END); while (it->Valid() && it->key().compare(ALL_END) < 0) { batch.Delete(it->key()); it->Next(); } // Note: any migration of settings for newer versions should happen here. // Write new schema version. batch.Put(SETTINGS_SCHEMA_VERSION, to_string(SCHEMA_VERSION)); s = db_->Write(write_options, &batch); throw_if_error(s, string("cannot clear DB after version mismatch, old version = ") + to_string(old_version) + ", new version = " + to_string(SCHEMA_VERSION)); stats_->num_entries_ = 0; stats_->hist_clear(); stats_->cache_size_ = 0; // init_stats() (called later) calls deserialize() on the stats, // so we need to create a proper stats record here. write_stats(); } } void PersistentStringCacheImpl::read_settings() { // Note: Loose error checking here. If someone deliberately // corrupts the settings table by writing garbage values for valid // keys, that's just too bad. We can't protect against Machiavelli. string val; auto s = db_->Get(read_options, SETTINGS_MAX_SIZE, &val); throw_if_error(s, "read_settings(): cannot read max size"); stats_->max_cache_size_ = stoll(val); s = db_->Get(read_options, SETTINGS_POLICY, &val); throw_if_error(s, "read_settings(): cannot read policy"); stats_->policy_ = static_cast(stoi(val)); } void PersistentStringCacheImpl::write_settings() { leveldb::WriteBatch batch; batch.Put(SETTINGS_MAX_SIZE, to_string(stats_->max_cache_size_)); batch.Put(SETTINGS_POLICY, to_string(static_cast(stats_->policy_))); auto s = db_->Write(write_options, &batch); throw_if_error(s, "write_settings()"); } void PersistentStringCacheImpl::read_stats() { string val; auto s = db_->Get(read_options, STATS_VALUES, &val); throw_if_error(s, "read_stats()"); stats_->deserialize(val); } void PersistentStringCacheImpl::write_stats() { auto s = db_->Put(write_options, STATS_VALUES, stats_->serialize()); throw_if_error(s, "write_stats()"); } bool PersistentStringCacheImpl::read_dirty_flag() const { string dirty; auto s = db_->Get(read_options, DIRTY_FLAG, &dirty); if (s.IsNotFound()) { return true; } return dirty != "0"; } void PersistentStringCacheImpl::write_dirty_flag(bool is_dirty) { auto s = db_->Put(write_options, DIRTY_FLAG, is_dirty ? "1" : "0"); throw_if_error(s, "write_dirty_flag()"); } PersistentStringCacheImpl::DataTuple PersistentStringCacheImpl::get_data(string const& key, bool& found) const { // mutex_ must be locked here! assert(key[0] == DATA_BEGIN[0]); string val; auto s = db_->Get(read_options, key, &val); throw_if_error(s, "get_data(): cannot read data"); if (!s.IsNotFound()) { found = true; return DataTuple(val); } found = false; return DataTuple(); } bool PersistentStringCacheImpl::get_value_and_metadata(string const& key, DataTuple& data, string& value, string* metadata) const { // mutex_ must be locked here! // Note: key is the un-prefixed key! string prefixed_key = k_data(key); IteratorUPtr it(db_->NewIterator(read_options)); it->Seek(prefixed_key); throw_if_error(it->status(), "get_value_and_metadata(): iterator error"); assert(it->Valid()); if (it->key() != leveldb::Slice(prefixed_key)) { return false; } data = DataTuple(it->value().ToString()); prefixed_key[0] = VALUES_BEGIN[0]; // Avoid string copy. it->Seek(prefixed_key); assert(it->Valid() && it->key().compare(prefixed_key) == 0); value = it->value().ToString(); if (metadata) { prefixed_key[0] = METADATA_BEGIN[0]; // Avoid string copy. it->Seek(prefixed_key); if (it->key().compare(prefixed_key) == 0) { *metadata = it->value().ToString(); } else { metadata->clear(); // Metadata may have been there previously, but isn't now. } } return true; } void PersistentStringCacheImpl::batch_delete(string const& key, DataTuple const& data, leveldb::WriteBatch& batch) { // mutex_ must be locked here! string prefixed_key = k_data(key); batch.Delete(prefixed_key); // Delete data. prefixed_key[0] = VALUES_BEGIN[0]; // Avoid string copy. batch.Delete(prefixed_key); // Delete value. prefixed_key[0] = METADATA_BEGIN[0]; // Avoid string copy. batch.Delete(prefixed_key); // Delete metadata batch.Delete(k_atime_index(data.atime, key)); // Delete atime index if (stats_->policy_ == CacheDiscardPolicy::lru_ttl) { string etime_key = k_etime_index(data.etime, key); batch.Delete(etime_key); } } void PersistentStringCacheImpl::delete_entry(string const& key, DataTuple const& data) { // mutex_ must be locked here! leveldb::WriteBatch batch; batch_delete(key, data, batch); auto s = db_->Write(write_options, &batch); throw_if_error(s, "delete_entry()"); // Update cache size and entries. stats_->hist_decrement(data.size); stats_->cache_size_ -= data.size; assert(stats_->cache_size_ >= 0); assert(stats_->cache_size_ <= stats_->max_cache_size_); --stats_->num_entries_; assert(stats_->num_entries_ >= 0); assert(stats_->cache_size_ == 0 || stats_->num_entries_ != 0); assert(stats_->num_entries_ == 0 || stats_->cache_size_ != 0); } void PersistentStringCacheImpl::delete_at_least(int64_t bytes_needed, string const& skip_key) { // mutex_ must be locked here! assert(bytes_needed > 0); assert(bytes_needed <= stats_->cache_size_); int64_t deleted_bytes = 0; int64_t deleted_entries = 0; leveldb::WriteBatch batch; // Step 1: Delete all expired entries. if (stats_->policy_ == CacheDiscardPolicy::lru_ttl) { auto now_time = now_ticks(); IteratorUPtr it(db_->NewIterator(read_options)); leveldb::Slice const etime_prefix(ETIME_BEGIN); it->Seek(etime_prefix); while (it->Valid()) { if (!it->key().starts_with(etime_prefix)) { break; } string etime_key = it->key().ToString(); TimeKeyTuple ek(etime_key.substr(1)); // Strip prefix to create the etime/key tuple. if (!skip_key.empty() && ek.key == skip_key) { // Too hard to hit with a test because the entry must expire // in between put_metadata() having decided that it's still // unexpired and now_time taken above. // LCOV_EXCL_START it->Next(); continue; // This entry must not be deleted (see put_metadata()). // LCOV_EXCL_STOP } if (ek.time > now_time) { break; // Anything past this point has not expired yet. } string prefixed_key = k_data(ek.key); string val; auto s = db_->Get(read_options, prefixed_key, &val); throw_if_error(s, "delete_at_least: cannot read data"); DataTuple dt(move(val)); int64_t size = stoll(it->value().ToString()); deleted_bytes += size; bytes_needed -= size; ++deleted_entries; batch_delete(ek.key, dt, batch); --stats_->num_entries_; ++stats_->ttl_evictions_; stats_->hist_decrement(size); stats_->cache_size_ -= size; call_handler(ek.key, CacheEventIndex::evict_ttl); it->Next(); } throw_if_error(it->status(), "delete_at_least(): expiry iterator error"); } // Close iterator. if (deleted_entries) { // Need to commit the batch here, otherwise what follows will not see the changes made above. auto s = db_->Write(write_options, &batch); throw_if_error(s, "delete_at_least(): expiry write error"); batch.Clear(); } // Step 2: If we still need more room, delete entries in LRU order until we have enough room. if (bytes_needed > 0) { // Run over the Atime index and delete in old-to-new order. IteratorUPtr it(db_->NewIterator(read_options)); leveldb::Slice const atime_prefix(ATIME_BEGIN); it->Seek(atime_prefix); while (it->Valid() && bytes_needed > 0 && it->key().starts_with(atime_prefix)) { TimeKeyTuple atk(it->key().ToString().substr(1)); // Strip prefix to create the atime/key tuple. if (!skip_key.empty() && atk.key == skip_key) { it->Next(); continue; // This entry must not be deleted (see put_metadata()). } int64_t size = stoll(it->value().ToString()); deleted_bytes += size; bytes_needed -= size; ++deleted_entries; string data_string; string prefixed_key = k_data(atk.key); auto s = db_->Get(read_options, prefixed_key, &data_string); assert(!s.IsNotFound()); throw_if_error(s, "delete_at_least()"); DataTuple dt(move(data_string)); batch_delete(atk.key, dt, batch); --stats_->num_entries_; ++stats_->lru_evictions_; stats_->hist_decrement(size); stats_->cache_size_ -= size; call_handler(atk.key, CacheEventIndex::evict_lru); it->Next(); } throw_if_error(it->status(), "delete_at_least(): LRU iterator error"); assert(deleted_bytes > 0); assert(bytes_needed <= 0); } auto s = db_->Write(write_options, &batch); throw_if_error(s, "delete_at_least(): LRU write error"); assert(stats_->cache_size_ >= 0); assert(stats_->num_entries_ >= 0); assert(stats_->cache_size_ == 0 || stats_->num_entries_ != 0); assert(stats_->num_entries_ == 0 || stats_->cache_size_ != 0); } void PersistentStringCacheImpl::call_handler(string const& key, CacheEventIndex event_index) const { // mutex_ must be locked here! typedef underlying_type::type IndexType; auto handler = handlers_[static_cast(event_index)]; if (handler) { try { IndexType index = static_cast(event_index); handler(key, static_cast(1 << index), stats_); } catch (...) { // Ignored } } } string PersistentStringCacheImpl::make_message(leveldb::Status const& s, string const& msg) const { return class_name + ": " + msg + ": " + s.ToString() + " (cache_path: " + stats_->cache_path_ + ")"; } string PersistentStringCacheImpl::make_message(string const& msg) const { return class_name + ": " + msg + " (cache_path: " + stats_->cache_path_ + ")"; } void PersistentStringCacheImpl::throw_if_error(leveldb::Status const& s, string const& msg) const { if (!s.ok() && !s.IsNotFound()) { if (s.IsCorruption()) { throw system_error(666, generic_category(), make_message(s, msg)); // LCOV_EXCL_LINE } throw runtime_error(make_message(s, msg)); } } void PersistentStringCacheImpl::throw_logic_error(string const& msg) const { throw logic_error(make_message(msg)); } void PersistentStringCacheImpl::throw_invalid_argument(string const& msg) const { throw invalid_argument(make_message(msg)); } void PersistentStringCacheImpl::throw_corrupt_error(string const& msg) const { throw system_error(666, generic_category(), make_message(msg)); } } // namespace internal } // namespace core persistent-cache-cpp-1.0.5/src/core/persistent_cache_stats.cpp000066400000000000000000000143431414021052300245170ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include using namespace std; namespace core { PersistentCacheStats::PersistentCacheStats() : p_(make_shared()) , internal_(false) { } /// @cond // So we can point at the cache's internal instance, which avoids a copy. PersistentCacheStats::PersistentCacheStats(shared_ptr const& p) noexcept : p_(p) , internal_(true) { } /// @endcond PersistentCacheStats::PersistentCacheStats(PersistentCacheStats const& other) : internal_(false) { if (other.internal_) { // Source is pointing at the internal instance, so we make a copy here to de-couple from it. p_ = make_shared(*other.p_); } else { p_ = other.p_; // Class is read-only, so we can just point at the same impl. } } PersistentCacheStats::PersistentCacheStats(PersistentCacheStats&& other) noexcept : internal_(false) { if (other.internal_) { // Source is pointing at the internal instance, so we make a copy here to de-couple from it. p_ = make_shared(*other.p_); } else { p_ = other.p_; // Move must leave the instance in a usable state, so we just copy the shared_ptr. } } PersistentCacheStats& PersistentCacheStats::operator=(PersistentCacheStats const& rhs) { if (this != &rhs) { if (rhs.internal_) { // Source is pointing at the internal instance, so we make a copy here to de-couple from it. p_ = make_shared(*rhs.p_); internal_ = false; } else { assert(internal_ == false); p_ = rhs.p_; // Class is read-only, so we can just point at the same impl. } } return *this; } PersistentCacheStats& PersistentCacheStats::operator=(PersistentCacheStats&& rhs) { if (rhs.internal_) { // Source is pointing at the internal instance, so we make a copy here to de-couple from it. p_ = make_shared(*rhs.p_); internal_ = false; } else { assert(internal_ == false); p_ = rhs.p_; // Move must leave the instance in a usable state, so we just copy the shared_ptr. } return *this; } string PersistentCacheStats::cache_path() const { return p_->cache_path_; } CacheDiscardPolicy PersistentCacheStats::policy() const noexcept { return p_->policy_; } int64_t PersistentCacheStats::size() const noexcept { return p_->num_entries_; } int64_t PersistentCacheStats::size_in_bytes() const noexcept { return p_->cache_size_; } int64_t PersistentCacheStats::max_size_in_bytes() const noexcept { return p_->max_cache_size_; } int64_t PersistentCacheStats::hits() const noexcept { return p_->hits_; } int64_t PersistentCacheStats::misses() const noexcept { return p_->misses_; } int64_t PersistentCacheStats::hits_since_last_miss() const noexcept { return p_->hits_since_last_miss_; } int64_t PersistentCacheStats::misses_since_last_hit() const noexcept { return p_->misses_since_last_hit_; } int64_t PersistentCacheStats::longest_hit_run() const noexcept { return p_->longest_hit_run_; } int64_t PersistentCacheStats::longest_miss_run() const noexcept { return p_->longest_miss_run_; } int64_t PersistentCacheStats::hit_runs() const noexcept { return p_->num_hit_runs_; } int64_t PersistentCacheStats::miss_runs() const noexcept { return p_->num_miss_runs_; } double PersistentCacheStats::avg_hit_run_length() const noexcept { return p_->num_hit_runs_ == 0 ? 0.0 : double(p_->hits_) / p_->num_hit_runs_; } double PersistentCacheStats::avg_miss_run_length() const noexcept { return p_->num_miss_runs_ == 0 ? 0.0 : double(p_->misses_) / p_->num_miss_runs_; } int64_t PersistentCacheStats::ttl_evictions() const noexcept { return p_->ttl_evictions_; } int64_t PersistentCacheStats::lru_evictions() const noexcept { return p_->lru_evictions_; } chrono::system_clock::time_point PersistentCacheStats::most_recent_hit_time() const noexcept { return p_->most_recent_hit_time_; } chrono::system_clock::time_point PersistentCacheStats::most_recent_miss_time() const noexcept { return p_->most_recent_miss_time_; } chrono::system_clock::time_point PersistentCacheStats::longest_hit_run_time() const noexcept { return p_->longest_hit_run_time_; } chrono::system_clock::time_point PersistentCacheStats::longest_miss_run_time() const noexcept { return p_->longest_miss_run_time_; } PersistentCacheStats::Histogram const& PersistentCacheStats::histogram() const noexcept { return p_->hist_; } PersistentCacheStats::HistogramBounds const& PersistentCacheStats::histogram_bounds() noexcept { static HistogramBounds bounds = []() { HistogramBounds b; b.push_back({1, 9}); // First bin collapses 1 - 9 into a single bin. unsigned i = 1; auto constexpr num_powers = (PersistentCacheStats::NUM_BINS - 2) / 9; for (; i <= num_powers; ++i) { for (unsigned int j = 1; j < 10; ++j) { uint64_t lower = pow(10, i) * j; uint64_t upper = lower + pow(10, i) - 1; b.push_back({lower, upper}); } } b.push_back({pow(10, i), std::numeric_limits::max()}); return b; }(); return bounds; } } // namespace core persistent-cache-cpp-1.0.5/src/core/persistent_string_cache.cpp000066400000000000000000000161021414021052300246620ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ // @cond #include #include #include using namespace std; namespace core { PersistentStringCache::PersistentStringCache(string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) : p_(new internal::PersistentStringCacheImpl(cache_path, max_size_in_bytes, policy, this)) { } PersistentStringCache::PersistentStringCache(string const& cache_path) : p_(new internal::PersistentStringCacheImpl(cache_path, this)) { } PersistentStringCache::PersistentStringCache(PersistentStringCache&&) = default; PersistentStringCache& PersistentStringCache::operator=(PersistentStringCache&&) = default; PersistentStringCache::~PersistentStringCache() = default; PersistentStringCache::UPtr PersistentStringCache::open(string const& cache_path, int64_t max_size_in_bytes, CacheDiscardPolicy policy) { return PersistentStringCache::UPtr(new PersistentStringCache(cache_path, max_size_in_bytes, policy)); } PersistentStringCache::UPtr PersistentStringCache::open(string const& cache_path) { return PersistentStringCache::UPtr(new PersistentStringCache(cache_path)); } Optional PersistentStringCache::get(string const& key) const { string value; return p_->get(key, value) ? Optional(move(value)) : Optional(); } Optional PersistentStringCache::get_data(string const& key) const { string value; string metadata; return p_->get(key, value, &metadata) ? Optional(move(Data{move(value), move(metadata)})) : Optional(); } Optional PersistentStringCache::get_metadata(string const& key) const { string metadata; return p_->get_metadata(key, metadata) ? Optional(move(metadata)) : Optional(); } bool PersistentStringCache::contains_key(string const& key) const { return p_->contains_key(key); } int64_t PersistentStringCache::size() const noexcept { return p_->size(); } int64_t PersistentStringCache::size_in_bytes() const noexcept { return p_->size_in_bytes(); } int64_t PersistentStringCache::max_size_in_bytes() const noexcept { return p_->max_size_in_bytes(); } int64_t PersistentStringCache::disk_size_in_bytes() const { return p_->disk_size_in_bytes(); } CacheDiscardPolicy PersistentStringCache::discard_policy() const noexcept { return p_->discard_policy(); } PersistentCacheStats PersistentStringCache::stats() const { return p_->stats(); } bool PersistentStringCache::put(string const& key, string const& value, chrono::time_point expiry_time) { return p_->put(key, value.data(), value.size(), nullptr, 0, expiry_time); } bool PersistentStringCache::put(string const& key, char const* value, int64_t size, chrono::time_point expiry_time) { return p_->put(key, value, size, nullptr, 0, expiry_time); } bool PersistentStringCache::put(string const& key, string const& value, string const& metadata, chrono::time_point expiry_time) { return p_->put(key, value.data(), value.size(), metadata.data(), metadata.size(), expiry_time); } bool PersistentStringCache::put(string const& key, char const* value, int64_t value_size, char const* metadata, int64_t metadata_size, chrono::time_point expiry_time) { return p_->put(key, value, value_size, metadata, metadata_size, expiry_time); } Optional PersistentStringCache::get_or_put( string const& key, PersistentStringCache::Loader const& load_func) { string value; bool found = p_->get_or_put(key, value, load_func); return found ? Optional(move(value)) : Optional(); } Optional PersistentStringCache::get_or_put_data( string const& key, PersistentStringCache::Loader const& load_func) { string value; string metadata; return p_->get_or_put(key, value, &metadata, load_func) ? Optional(Data{move(value), move(metadata)}) : Optional(); } bool PersistentStringCache::put_metadata(string const& key, string const& metadata) { return p_->put_metadata(key, metadata.data(), metadata.size()); } bool PersistentStringCache::put_metadata(string const& key, char const* metadata, int64_t size) { return p_->put_metadata(key, metadata, size); } Optional PersistentStringCache::take(string const& key) { string value; return p_->take(key, value) ? Optional(move(value)) : Optional(); } Optional PersistentStringCache::take_data(string const& key) { string value; string metadata; return p_->take(key, value, &metadata) ? Optional(move(Data{move(value), move(metadata)})) : Optional(); } bool PersistentStringCache::invalidate(string const& key) { return p_->invalidate(key); } void PersistentStringCache::invalidate(vector const& keys) { p_->invalidate(keys); } void PersistentStringCache::invalidate(initializer_list const& keys) { invalidate(keys.begin(), keys.end()); } void PersistentStringCache::invalidate() { p_->invalidate(); } bool PersistentStringCache::touch(string const& key, chrono::time_point expiry_time) { return p_->touch(key, expiry_time); } void PersistentStringCache::clear_stats() { p_->clear_stats(); } void PersistentStringCache::resize(int64_t size_in_bytes) { p_->resize(size_in_bytes); } void PersistentStringCache::trim_to(int64_t used_size_in_bytes) { p_->trim_to(used_size_in_bytes); } void PersistentStringCache::compact() { p_->compact(); } void PersistentStringCache::set_handler(CacheEvent events, EventCallback cb) { p_->set_handler(events, cb); } } // namespace core // @endcond persistent-cache-cpp-1.0.5/tests/000077500000000000000000000000001414021052300166705ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/CMakeLists.txt000066400000000000000000000004261414021052300214320ustar00rootroot00000000000000add_subdirectory(copyright) add_subdirectory(core) add_subdirectory(headers) add_subdirectory(whitespace) # Tests in subdirectories set this. We push it up to the parent so we can # exclude the unit tests from coverage. set(UNIT_TEST_TARGETS ${UNIT_TEST_TARGETS} PARENT_SCOPE) persistent-cache-cpp-1.0.5/tests/copyright/000077500000000000000000000000001414021052300207005ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/copyright/CMakeLists.txt000066400000000000000000000002551414021052300234420ustar00rootroot00000000000000# # Test that all source files contain a copyright header. # add_test(copyright ${CMAKE_CURRENT_SOURCE_DIR}/check_copyright.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) persistent-cache-cpp-1.0.5/tests/copyright/check_copyright.sh000077500000000000000000000032751414021052300244130ustar00rootroot00000000000000#!/bin/sh # Copyright (C) 2013 Canonical Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Authored by: Michi Henning # # Check that we have acceptable license information in our source files. # usage() { echo "usage: check_copyright dir [ignore_dir]" >&2 exit 2 } [ $# -lt 1 ] && usage [ $# -gt 2 ] && usage # TODO: Temporary hack to work around broken licensecheck on xenial. Remove this once that is fixed. distro=$(lsb_release -c -s) [ "$distro" = "xenial" ] && { exit 0 } ignore_pat="\\.gitignore|\\.git/.*|\\.pc/.*|\\.sci$|debian|HACKING|README|\\.txt$|\\.in$|\\.pm$" # # We don't use the -i option of licensecheck to add ignore_dir to the pattern because Jenkins creates directories # with names that contain regex meta-characters, such as "." and "+". Instead, if ingnore_dir is set, we post-filter # the output with grep -F, so we don't get false positives from licensecheck. # [ $# -eq 2 ] && ignore_dir="$2" if [ -n "$ignore_dir" ] then licensecheck -i "$ignore_pat" -r "$1" | grep -F "$ignore_dir" -v | grep 'No copyright' else licensecheck -i "$ignore_pat" -r "$1" | grep 'No copyright' fi [ $? -eq 0 ] && exit 1 exit 0 persistent-cache-cpp-1.0.5/tests/core/000077500000000000000000000000001414021052300176205ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/core/CMakeLists.txt000066400000000000000000000005221414021052300223570ustar00rootroot00000000000000include(FindPkgConfig) find_package(GMock) find_package(Boost COMPONENTS system filesystem) set(TESTLIBS ${LIBNAME} boost_filesystem boost_system leveldb ${GMOCK_LIBRARIES}) add_subdirectory(persistent_cache) add_subdirectory(persistent_string_cache) add_subdirectory(internal) set(UNIT_TEST_TARGETS ${UNIT_TEST_TARGETS} PARENT_SCOPE) persistent-cache-cpp-1.0.5/tests/core/internal/000077500000000000000000000000001414021052300214345ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/core/internal/CMakeLists.txt000066400000000000000000000001641414021052300241750ustar00rootroot00000000000000add_subdirectory(persistent_string_cache_impl) set(UNIT_TEST_TARGETS ${UNIT_TEST_TARGETS} ${TARGETS} PARENT_SCOPE) persistent-cache-cpp-1.0.5/tests/core/internal/persistent_string_cache_impl/000077500000000000000000000000001414021052300273665ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/core/internal/persistent_string_cache_impl/CMakeLists.txt000066400000000000000000000006401414021052300321260ustar00rootroot00000000000000add_executable(persistent_string_cache_impl_test persistent_string_cache_impl_test.cpp) target_link_libraries(persistent_string_cache_impl_test ${TESTLIBS}) add_definitions(-DTEST_DIR="${CMAKE_CURRENT_BINARY_DIR}") add_test(persistent_string_cache_impl persistent_string_cache_impl_test) set(TARGETS ${TARGETS} persistent_string_cache_impl_test) set(UNIT_TEST_TARGETS ${UNIT_TEST_TARGETS} ${TARGETS} PARENT_SCOPE) persistent_string_cache_impl_test.cpp000066400000000000000000002042331414021052300370100ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/core/internal/persistent_string_cache_impl/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wctor-dtor-privacy" #include #pragma GCC diagnostic pop #include #include #include #include // https://stackoverflow.com/a/9158263 #define RESET "\033[0m" #define BOLDYELLOW "\033[1m\033[33m" /* Bold Yellow */ using namespace std; using namespace core; using namespace core::internal; // Removes the contents of db_dir, but not db_dir itself. void unlink_db(string const& db_dir) { namespace fs = boost::filesystem; try { for (fs::directory_iterator end, it(db_dir); it != end; ++it) { remove_all(it->path()); } } catch (...) { } } const string TEST_DB = TEST_DIR "/db"; TEST(PersistentStringCacheImpl, basic) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024 * 1024, CacheDiscardPolicy::lru_ttl); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); EXPECT_FALSE(c.contains_key("hello")); string val; EXPECT_TRUE(c.put("e", "")); // Empty value EXPECT_EQ(1, c.size()); EXPECT_EQ(1, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("e")); EXPECT_TRUE(c.get("e", val)); EXPECT_EQ("", val); EXPECT_FALSE(c.contains_key("no such key")); EXPECT_FALSE(c.invalidate("no such key")); EXPECT_FALSE(c.get("no such key", val)); EXPECT_FALSE(c.take("no such key", val)); EXPECT_TRUE(c.take("e", val)); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); EXPECT_FALSE(c.contains_key("e")); EXPECT_EQ("", val); EXPECT_TRUE(c.put("hello", "world", strlen("world"))); // Different put, for coverage EXPECT_EQ(1, c.size()); EXPECT_EQ(10, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("hello")); EXPECT_TRUE(c.get("hello", val)); EXPECT_EQ("world", val); EXPECT_TRUE(c.invalidate("hello")); EXPECT_EQ(0, c.size()); EXPECT_FALSE(c.contains_key("hello")); c.put("k1", "v1"); EXPECT_EQ(1, c.size()); EXPECT_EQ(4, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("k1")); EXPECT_TRUE(c.get("k1", val)); EXPECT_EQ("v1", val); c.put("k2", "v2"); EXPECT_EQ(2, c.size()); EXPECT_EQ(8, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("k2")); EXPECT_TRUE(c.get("k2", val)); EXPECT_EQ("v2", val); EXPECT_TRUE(c.get("k1", val)); EXPECT_EQ("v1", val); c.invalidate(); EXPECT_FALSE(c.contains_key("k1")); EXPECT_FALSE(c.contains_key("k2")); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); c.put("k1", "v1"); c.put("k2", "v2"); EXPECT_TRUE(c.contains_key("k1")); EXPECT_TRUE(c.contains_key("k2")); EXPECT_TRUE(c.invalidate("k2")); EXPECT_TRUE(c.contains_key("k1")); EXPECT_FALSE(c.contains_key("k2")); EXPECT_TRUE(c.invalidate("k1")); EXPECT_FALSE(c.contains_key("k1")); EXPECT_FALSE(c.contains_key("k2")); c.put("k1", "v1"); c.put("k2", "v2"); EXPECT_TRUE(c.contains_key("k1")); EXPECT_TRUE(c.contains_key("k2")); EXPECT_TRUE(c.invalidate("k1")); EXPECT_FALSE(c.contains_key("k1")); EXPECT_TRUE(c.contains_key("k2")); EXPECT_TRUE(c.invalidate("k2")); EXPECT_FALSE(c.contains_key("k1")); EXPECT_FALSE(c.contains_key("k2")); c.put("k1", "v1"); EXPECT_TRUE(c.contains_key("k1")); EXPECT_TRUE(c.take("k1", val)); EXPECT_EQ("v1", val); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); EXPECT_FALSE(c.contains_key("k1")); EXPECT_EQ("v1", val); val = "newval"; EXPECT_FALSE(c.take("k1", val)); EXPECT_EQ("newval", val); // Already-expired entries are not added. EXPECT_FALSE(c.put("expired", "val", chrono::system_clock::now() - chrono::seconds(1))); EXPECT_FALSE(c.contains_key("expired")); // Non-expired entries are added. EXPECT_TRUE(c.put("not expired", "val", chrono::system_clock::now() + chrono::seconds(5))); EXPECT_TRUE(c.contains_key("not expired")); // Non-expired entries are refreshed. EXPECT_TRUE(c.put("not expired", "val", chrono::system_clock::now() + chrono::seconds(3))); EXPECT_TRUE(c.contains_key("not expired")); // Remove non-existent key EXPECT_FALSE(c.contains_key("x")); EXPECT_FALSE(c.invalidate("x")); EXPECT_FALSE(c.contains_key("x")); // Add a key twice with same value { c.invalidate(); string const in_val("X"); { string out_val; EXPECT_TRUE(c.put("x", in_val)); EXPECT_EQ(1, c.size()); EXPECT_TRUE(c.get("x", out_val)); EXPECT_EQ(in_val, out_val); } { string out_val; EXPECT_TRUE(c.put("x", in_val)); EXPECT_EQ(1, c.size()); EXPECT_TRUE(c.get("x", out_val)); EXPECT_EQ(in_val, out_val); } } // Add a key twice with different value { c.invalidate(); EXPECT_FALSE(c.contains_key("x")); string const val1 = "x"; string out_val; EXPECT_TRUE(c.put("x", val1)); EXPECT_EQ(1, c.size()); EXPECT_TRUE(c.get("x", out_val)); EXPECT_EQ(val1, out_val); string const val2 = "xy"; EXPECT_TRUE(c.put("x", val2)); EXPECT_EQ(1, c.size()); EXPECT_TRUE(c.get("x", out_val)); EXPECT_EQ(val2, out_val); } // touch() for a key that isn't there (for coverage) EXPECT_FALSE(c.touch("no_such_key")); // touch() with already-expired expiry time auto expiry_time = chrono::system_clock::now() - chrono::milliseconds(1); EXPECT_FALSE(c.touch("x", expiry_time)); // touch() with OK expiry time expiry_time = chrono::system_clock::now() + chrono::milliseconds(1000); EXPECT_TRUE(c.touch("x", expiry_time)); } TEST(PersistentStringCacheImpl, update) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_only); string new_val; string val(899, '1'); // Large value, near size limit EXPECT_TRUE(c.put("1", val)); EXPECT_EQ(1, c.size()); EXPECT_EQ(900, c.size_in_bytes()); EXPECT_TRUE(c.get("1", new_val)); EXPECT_EQ(val, new_val); string val2(99, '2'); // Second value, just fits EXPECT_TRUE(c.put("2", val2)); EXPECT_EQ(1000, c.size_in_bytes()); // Second value must be there EXPECT_TRUE(c.get("2", new_val)); EXPECT_EQ(val2, new_val); // First value is now the oldest value. val = string(1023, 'n'); // Size limit of cache EXPECT_TRUE(c.put("1", val)); // Replace the old value EXPECT_EQ(1024, c.size_in_bytes()); // Size must be at limit now EXPECT_TRUE(c.get("1", new_val)); // Old value must be there... EXPECT_EQ(val, new_val); // ... with correct contents. EXPECT_FALSE(c.contains_key("2")); // Old value must have evicted smaller newer value val = string(899, 'v'); // Make the value smaller EXPECT_TRUE(c.put("1", val)); // Replace the value EXPECT_EQ(900, c.size_in_bytes()); val2 = string(99, '2'); // Second value, just fits EXPECT_TRUE(c.put("2", val2)); // Add it EXPECT_EQ(1000, c.size_in_bytes()); // Check new size // First value is now the oldest value. string meta(124, 'm'); // Adding this fills cache to limit EXPECT_TRUE(c.put_metadata("1", meta)); // Add metadata EXPECT_EQ(1024, c.size_in_bytes()); // Size must be at limit now string new_meta; EXPECT_TRUE(c.get("1", new_val, &new_meta)); // Old value must be there... EXPECT_EQ(val, new_val); // ... with the right value... EXPECT_EQ(meta, new_meta); // ... and the right metadata } TEST(PersistentStringCacheImpl, metadata) { { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024 * 1024, CacheDiscardPolicy::lru_ttl); string metadata = "md"; EXPECT_FALSE(c.get_metadata("no_such_key", metadata)); EXPECT_EQ("md", metadata); // Original value must be intact after failure. c.put("1", "1"); EXPECT_FALSE(c.get_metadata("1", metadata)); EXPECT_EQ("md", metadata); // Original value must be intact after falure. c.put("1", "1", &metadata); EXPECT_TRUE(c.get_metadata("1", metadata)); EXPECT_EQ("md", metadata); EXPECT_EQ(4, c.size_in_bytes()); string val; EXPECT_TRUE(c.get("1", val, &metadata)); EXPECT_EQ("1", val); EXPECT_EQ("md", metadata); EXPECT_EQ(4, c.size_in_bytes()); val = ""; metadata = "xxx"; EXPECT_TRUE(c.put("1", "2")); EXPECT_TRUE(c.get("1", val, &metadata)); EXPECT_EQ("2", val); EXPECT_EQ("", metadata); // Previous metadata must have been removed. EXPECT_EQ(2, c.size_in_bytes()); val = ""; metadata = "md"; EXPECT_TRUE(c.put("1", "2", &metadata)); EXPECT_EQ(4, c.size_in_bytes()); val = ""; metadata = ""; EXPECT_TRUE(c.take("1", val, &metadata)); EXPECT_FALSE(c.get("1", val, &metadata)); EXPECT_EQ("2", val); EXPECT_EQ("md", metadata); EXPECT_EQ(0, c.size_in_bytes()); auto now = chrono::system_clock::now(); auto later = now + chrono::milliseconds(200); metadata = "md"; c.put("1", "a", &metadata, later); while (chrono::system_clock::now() <= later) { this_thread::sleep_for(chrono::milliseconds(5)); } metadata = "x"; EXPECT_FALSE(c.get_metadata("1", metadata)); // Expired entries don't return user data. EXPECT_EQ(4, c.size_in_bytes()); // Entry is still there, but invisible. EXPECT_TRUE(c.put("1", "")); // Replace expired entry with non-expiring one. EXPECT_EQ(1, c.size_in_bytes()); EXPECT_FALSE(c.get_metadata("1", metadata)); EXPECT_TRUE(c.put_metadata("1", "")); EXPECT_EQ(1, c.size_in_bytes()); EXPECT_TRUE(c.get_metadata("1", metadata)); EXPECT_EQ("", metadata); EXPECT_TRUE(c.put_metadata("1", "1")); EXPECT_EQ(2, c.size_in_bytes()); EXPECT_TRUE(c.get_metadata("1", metadata)); EXPECT_EQ("1", metadata); EXPECT_FALSE(c.put_metadata("no_such_key", "1")); EXPECT_EQ(2, c.size_in_bytes()); later = chrono::system_clock::now() + chrono::milliseconds(200); EXPECT_TRUE(c.put("1", "", later)); // Replace entry with expiring one. EXPECT_TRUE(c.put_metadata("1", "23")); // Not expired yet, must work. EXPECT_EQ(3, c.size_in_bytes()); while (chrono::system_clock::now() <= later) { this_thread::sleep_for(chrono::milliseconds(5)); } EXPECT_FALSE(c.put_metadata("1", "23")); // Expired now. EXPECT_EQ(3, c.size_in_bytes()); // Entry is still there, but invisible. } { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 100, CacheDiscardPolicy::lru_ttl); EXPECT_TRUE(c.put("1", "")); // 1-byte entry that we'll add metadata to later. this_thread::sleep_for(chrono::milliseconds(2)); // Make sure we get different timestamps. string val(44, 'a'); EXPECT_TRUE(c.put("2", val)); // 54 bytes of room left now. this_thread::sleep_for(chrono::milliseconds(2)); // Make sure we get different timestamps. EXPECT_TRUE(c.put("3", val)); // 9 bytes of room left now. EXPECT_EQ(91, c.size_in_bytes()); // "1" is oldest entry now. Try and add 45 bytes of metadata to it. // That must evict entry "2", which is the second-oldest, and leave // entry "3" intact. val += 'a'; ASSERT_EQ(45u, val.size()); EXPECT_TRUE(c.put_metadata("1", val)); EXPECT_FALSE(c.contains_key("2")); EXPECT_TRUE(c.contains_key("3")); EXPECT_EQ(91, c.size_in_bytes()); string md; EXPECT_TRUE(c.get_metadata("1", md)); EXPECT_EQ(val, md); } } TEST(PersistentStringCacheImpl, batch_invalidate) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024 * 1024, CacheDiscardPolicy::lru_ttl); c.put("a", ""); EXPECT_EQ(1, c.size()); c.invalidate({}); // Empty list EXPECT_EQ(1, c.size()); c.invalidate({""}); // Empty key EXPECT_EQ(1, c.size()); c.invalidate({"no_such_key"}); // Non-existent key EXPECT_EQ(1, c.size()); c.invalidate({"", "no_such_key"}); // Empty and non-existent key EXPECT_EQ(1, c.size()); c.invalidate({"a"}); // Existing key EXPECT_EQ(0, c.size()); c.put("a", ""); c.put("b", ""); c.put("c", ""); c.invalidate({"c", "", "x", "a"}); // Two existing keys, plus empty and non-existing keys EXPECT_EQ(1, c.size()); EXPECT_TRUE(c.contains_key("b")); } TEST(PersistentStringCacheImpl, get_or_put) { unlink_db(TEST_DB); // Need to use PersistentStringCache::open() here because the implementation needs // back-pointer to the pimpl. auto c = PersistentStringCache::open(TEST_DB, 1024 * 1024, CacheDiscardPolicy::lru_ttl); bool throw_std_exception_called; auto throw_std_exception = [&throw_std_exception_called](string const&, PersistentStringCache&) { throw_std_exception_called = true; throw runtime_error("std exception loader"); }; bool throw_unknown_exception_called; auto throw_unknown_exception = [&throw_unknown_exception_called](string const&, PersistentStringCache&) { throw_unknown_exception_called = true; throw 42; }; bool load_entry_called; auto load_entry = [&load_entry_called](string const& key, PersistentStringCache& c) { load_entry_called = true; EXPECT_TRUE(c.put(key, "load_entry")); }; bool load_with_metadata_called; auto load_with_metadata = [&load_with_metadata_called](string const& key, PersistentStringCache& c) { load_with_metadata_called = true; EXPECT_TRUE(c.put(key, "value", "metadata")); }; bool no_load_called; auto no_load = [&no_load_called](string const&, PersistentStringCache&) { no_load_called = true; // Do nothing. }; PersistentCacheStats s; c->put("1", "x"); s = c->stats(); EXPECT_EQ(0, s.hits()); throw_std_exception_called = false; EXPECT_TRUE(bool(c->get_or_put("1", throw_std_exception))); EXPECT_FALSE(throw_std_exception_called); // Entry exists, loader must not have run. s = c->stats(); EXPECT_EQ(1, s.hits()); EXPECT_EQ(0, s.misses()); c->invalidate(); EXPECT_EQ(0, c->size()); c->clear_stats(); throw_std_exception_called = false; try { c->get_or_put("1", throw_std_exception); FAIL(); } catch (runtime_error const& e) { EXPECT_EQ( "PersistentStringCache: get_or_put(): load_func exception: std exception loader " "(cache_path: " + TEST_DB + ")", e.what()); } s = c->stats(); EXPECT_EQ(0, s.hits()); EXPECT_EQ(1, s.misses()); c->clear_stats(); throw_unknown_exception_called = false; try { c->get_or_put("1", throw_unknown_exception); FAIL(); } catch (runtime_error const& e) { EXPECT_EQ( "PersistentStringCache: get_or_put(): load_func: unknown exception " "(cache_path: " + TEST_DB + ")", e.what()); } EXPECT_EQ(0, s.hits()); EXPECT_EQ(1, s.misses()); // Successful load without metadata. c->clear_stats(); load_entry_called = false; auto v = c->get_or_put("1", load_entry); EXPECT_TRUE(load_entry_called); EXPECT_TRUE(bool(v)); EXPECT_EQ("load_entry", *v); EXPECT_EQ(0, s.hits()); EXPECT_EQ(1, s.misses()); c->invalidate(); // Successful load with metadata. c->clear_stats(); load_with_metadata_called = false; auto data = c->get_or_put_data("1", load_with_metadata); EXPECT_TRUE(load_with_metadata_called); EXPECT_TRUE(bool(data)); EXPECT_EQ("value", data->value); EXPECT_EQ("metadata", data->metadata); EXPECT_EQ(0, s.hits()); EXPECT_EQ(1, s.misses()); c->invalidate(); // Unsuccessful load without error. c->clear_stats(); no_load_called = false; data = c->get_or_put_data("1", no_load); EXPECT_TRUE(no_load_called); EXPECT_FALSE(data); EXPECT_EQ(0, s.hits()); EXPECT_EQ(1, s.misses()); // Invalid key. try { c->get_or_put("", load_entry); // Empty key FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: get_or_put(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } } TEST(PersistentStringCacheImpl, open) { { unlink_db(TEST_DB); PersistentStringCacheImpl(TEST_DB, 666, CacheDiscardPolicy::lru_only); } { PersistentStringCacheImpl c(TEST_DB); EXPECT_EQ(666, c.max_size_in_bytes()); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); EXPECT_EQ(CacheDiscardPolicy::lru_only, c.discard_policy()); c.put("hello", "world"); EXPECT_EQ(1, c.size()); EXPECT_EQ(10, c.size_in_bytes()); } { PersistentStringCacheImpl c(TEST_DB); EXPECT_EQ(666, c.max_size_in_bytes()); EXPECT_EQ(1, c.size()); EXPECT_EQ(10, c.size_in_bytes()); EXPECT_EQ(CacheDiscardPolicy::lru_only, c.discard_policy()); } { PersistentStringCacheImpl c(TEST_DB); EXPECT_EQ(666, c.max_size_in_bytes()); EXPECT_EQ(1, c.size()); EXPECT_EQ(10, c.size_in_bytes()); EXPECT_EQ(CacheDiscardPolicy::lru_only, c.discard_policy()); c.resize(999); EXPECT_EQ(999, c.max_size_in_bytes()); } { PersistentStringCacheImpl c(TEST_DB); EXPECT_EQ(999, c.max_size_in_bytes()); EXPECT_EQ(1, c.size()); EXPECT_EQ(10, c.size_in_bytes()); EXPECT_EQ(CacheDiscardPolicy::lru_only, c.discard_policy()); } } TEST(PersistentStringCacheImpl, trim_to) { // Check that expired entries are deleted first. { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 3 * 1024, CacheDiscardPolicy::lru_ttl); auto now = chrono::system_clock::now(); auto later = now + chrono::milliseconds(100); string b(1023, 'x'); c.put("a", b); // 1024 bytes, don't expire c.put("b", b); // 1024 bytes, don't expire c.put("c", b, later); // 1024 bytes, expire while (chrono::system_clock::now() <= later) { this_thread::sleep_for(chrono::milliseconds(5)); } EXPECT_EQ(3 * 1024, c.size_in_bytes()); c.trim_to(2 * 1024); EXPECT_EQ(2, c.size()); EXPECT_EQ(2 * 1024, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("a")); EXPECT_TRUE(c.contains_key("b")); EXPECT_FALSE(c.contains_key("c")); // trim_to(2 * 1024) must have deleted expired record only c.trim_to(500); // Less than the last remaining record EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); EXPECT_EQ(3 * 1024, c.max_size_in_bytes()); } // Check that expired entries are deleted first, followed by other entries. { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 3 * 1024, CacheDiscardPolicy::lru_ttl); auto now = chrono::system_clock::now(); auto later = now + chrono::milliseconds(100); string b(1023, 'x'); c.put("a", b); // 1024 bytes, don't expire c.put("b", b, later); // 1024 bytes, expire while (chrono::system_clock::now() < later) { this_thread::sleep_for(chrono::milliseconds(5)); } c.put("c", b); // 1024 bytes, don't expire c.trim_to(1024); // Remove two records EXPECT_EQ(1, c.size()); EXPECT_EQ(1024, c.size_in_bytes()); EXPECT_FALSE(c.contains_key("a")); // trim_to(1024) must have deleted older record; EXPECT_FALSE(c.contains_key("b")); // trim_to(1024) must have deleted expired record EXPECT_TRUE(c.contains_key("c")); } // Check that, when reaping expired entries, we don't delete too many records. { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 3 * 1024, CacheDiscardPolicy::lru_ttl); auto now = chrono::system_clock::now(); auto later = now + chrono::milliseconds(100); auto much_later = now + chrono::milliseconds(200); string b(1023, 'x'); c.put("a", b, much_later); // 1024 bytes, expire second c.put("b", b, later); // 1024 bytes, expire first c.put("c", b); // 1024 bytes, don't expire while (chrono::system_clock::now() < later) { this_thread::sleep_for(chrono::milliseconds(5)); } c.trim_to(2048); // Remove one record EXPECT_EQ(2, c.size()); EXPECT_EQ(2048, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("a")); // trim_to(2048) must have kept that record EXPECT_FALSE(c.contains_key("b")); // trim_to(2048) must have deleted expired record EXPECT_TRUE(c.contains_key("c")); // trim_to(2048) must kept non-expiring record } // Check that non-expired entries are not deleted. { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 3 * 1024, CacheDiscardPolicy::lru_ttl); auto now = chrono::system_clock::now(); auto later = now + chrono::milliseconds(200); string b(1023, 'x'); c.put("a", b, later); // 1024 bytes, expire in 200 ms this_thread::sleep_for(chrono::milliseconds(50)); c.put("b", b); // 1024 bytes, don't expire this_thread::sleep_for(chrono::milliseconds(100)); c.put("c", b); // 1024 bytes, don't expire EXPECT_EQ(3, c.size()); EXPECT_EQ(3 * 1024, c.size_in_bytes()); c.trim_to(1024); // Remove two records EXPECT_EQ(1, c.size()); EXPECT_EQ(1024, c.size_in_bytes()); EXPECT_EQ(3 * 1024, c.max_size_in_bytes()); EXPECT_FALSE(c.contains_key("a")); // a doesn't expire, but is the oldest record EXPECT_FALSE(c.contains_key("b")); // b is the second oldest EXPECT_TRUE(c.contains_key("c")); // c is the newest, must still be there } // Check that get() and touch() update the access time. { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 3 * 1024, CacheDiscardPolicy::lru_ttl); string b(1023, 'x'); c.put("a", b); this_thread::sleep_for(chrono::milliseconds(10)); c.put("b", b); this_thread::sleep_for(chrono::milliseconds(10)); c.put("c", b); this_thread::sleep_for(chrono::milliseconds(10)); string out_val; c.get("a", out_val); // a is most-recently used entry c.trim_to(2 * 1024); // Leave two records EXPECT_EQ(2, c.size()); EXPECT_EQ(2 * 1024, c.size_in_bytes()); EXPECT_EQ(3 * 1024, c.max_size_in_bytes()); EXPECT_TRUE(c.contains_key("a")); // a is the newest EXPECT_FALSE(c.contains_key("b")); // b is the oldest EXPECT_TRUE(c.contains_key("c")); // c is the second oldest // Prevent touch from happening in the same millisecond as the last get(). this_thread::sleep_for(chrono::milliseconds(10)); EXPECT_TRUE(c.touch("c")); // a is now the oldest c.trim_to(1 * 1024); // Leave only one record EXPECT_EQ(1, c.size()); EXPECT_EQ(1 * 1024, c.size_in_bytes()); EXPECT_FALSE(c.contains_key("a")); EXPECT_FALSE(c.contains_key("b")); EXPECT_TRUE(c.contains_key("c")); // Check that trim_to(0) works. c.trim_to(0); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); } } TEST(PersistentStringCacheImpl, policy_get_and_contains) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 10 * 1024, CacheDiscardPolicy::lru_ttl); string b(20, 'x'); string out_val; // Check that retrieval of non-expired entry works irrespective of policy. auto expiry_time = chrono::system_clock::now() + chrono::milliseconds(200); c.put("x", b, expiry_time); EXPECT_TRUE(c.get("x", out_val)); EXPECT_EQ(b, out_val); // Let the entry expire. It must be invisible, but still uses space. this_thread::sleep_for(chrono::milliseconds(210)); EXPECT_FALSE(c.contains_key("x")); EXPECT_FALSE(c.get("x", out_val)); EXPECT_EQ(1, c.size()); EXPECT_EQ(21, c.size_in_bytes()); // Removing the entry must pretend that it wasn't there, but will actually remove it. EXPECT_FALSE(c.invalidate("x")); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); } TEST(PersistentStringCacheImpl, policy_take_lru_ttl) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 10 * 1024, CacheDiscardPolicy::lru_ttl); string b(20, 'x'); auto expiry_time = chrono::system_clock::now() + chrono::milliseconds(100); c.put("x", b, expiry_time); EXPECT_EQ(21, c.size_in_bytes()); // Let the entry expire. this_thread::sleep_for(chrono::milliseconds(110)); EXPECT_EQ(1, c.size()); EXPECT_EQ(21, c.size_in_bytes()); // take() must fail to remove the entry because policy is lru_ttl. string out_val; EXPECT_FALSE(c.take("x", out_val)); EXPECT_FALSE(c.contains_key("x")); // And the entry must have been physically removed regardless. EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); } // Note: get_or_put() exceptions are tested by the get_and_put test. TEST(PersistentStringCacheImpl, exceptions) { unlink_db(TEST_DB); // Open with different size. { PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_ttl); } try { PersistentStringCacheImpl c(TEST_DB, 2048, CacheDiscardPolicy::lru_ttl); FAIL(); } catch (logic_error const& e) { EXPECT_EQ( "PersistentStringCache: existing cache opened with different max_size_in_bytes (2048), " "existing size = 1024 (cache_path: " + TEST_DB + ")", e.what()); } // Open with different policy. try { PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_only); FAIL(); } catch (logic_error const& e) { EXPECT_EQ( "PersistentStringCache: existing cache opened with different policy (lru_only), " "existing policy = lru_ttl (cache_path: " + TEST_DB + ")", e.what()); } // Open non-existent cache try { PersistentStringCacheImpl c("no_such_cache"); FAIL(); } catch (runtime_error const& e) { EXPECT_STREQ( "PersistentStringCache: cannot open or create cache: Invalid argument: no_such_cache: " "does not exist (create_if_missing is false) (cache_path: no_such_cache)", e.what()); } // Invalid size argument try { PersistentStringCacheImpl c(TEST_DB, 0, CacheDiscardPolicy::lru_ttl); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ( "PersistentStringCache: invalid max_size_in_bytes (0): " "value must be > 0 (cache_path: " + TEST_DB + ")", e.what()); } // Database file not writable if (geteuid() != 0) { ASSERT_EQ(0, system(string("chmod 000 " + TEST_DB).c_str())); try { PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_ttl); FAIL(); } catch (runtime_error const& e) { ASSERT_EQ(0, system(string("chmod 777 " + TEST_DB).c_str())); string msg = e.what(); EXPECT_TRUE(boost::starts_with(msg, "PersistentStringCache: cannot open or create cache: ")) << msg; } ASSERT_EQ(0, system(string("chmod 777 " + TEST_DB).c_str())); } else { std::cout << BOLDYELLOW << "WARNING: " << RESET "Test runs as root. The permission denied test is skipped. Fix the build system.\n"; } // Record too large try { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_ttl); { string key("a"); string b(1023, 'b'); c.put(key, b); // OK, exactly 1 KB c.invalidate(key); } string key("a"); string b(1024, 'b'); c.put(key, b); // Not OK, 1 KB plus one byte FAIL(); } catch (logic_error const& e) { EXPECT_EQ( "PersistentStringCache: put(): cannot add 1025-byte record to " "cache with maximum size of 1024 (cache_path: " + TEST_DB + ")", e.what()); } { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_ttl); // trim_to() with negative size try { c.trim_to(-1); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ( "PersistentStringCache: trim_to(): invalid used_size_in_bytes (-1): " "value must be >= 0 (cache_path: " + TEST_DB + ")", e.what()); } // trim_to() with excessive size try { c.trim_to(1025); FAIL(); } catch (logic_error const& e) { EXPECT_EQ( "PersistentStringCache: trim_to(): invalid used_size_in_bytes (1025): " "value must be <= max_size_in_bytes (1024) (cache_path: " + TEST_DB + ")", e.what()); } // resize() with invalid size try { c.resize(0); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ( "PersistentStringCache: resize(): invalid size_in_bytes (0): " "value must be > 0 (cache_path: " + TEST_DB + ")", e.what()); } // Open non-existent DB try { PersistentStringCacheImpl(TEST_DIR "/no_such_db"); FAIL(); } catch (runtime_error const& e) { string msg = e.what(); EXPECT_TRUE(boost::starts_with(msg, "PersistentStringCache: cannot open or create cache: ")) << msg; } // get() with empty key string out_val = "x"; try { c.get("", out_val); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("x", out_val); EXPECT_EQ("PersistentStringCache: get(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // contains_key() with empty key try { string out_val; c.contains_key(""); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: contains_key(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // put() with empty key try { c.put("", "val"); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: put(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // put() with nullptr value try { c.put("1", nullptr, 10); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: put(): value must not be nullptr (cache_path: " + TEST_DB + ")", e.what()); } // put() with negative size try { c.put("1", "md", -1); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: put(): invalid negative value size: -1 (cache_path: " + TEST_DB + ")", e.what()); } // put() with negative metadata size try { c.put("1", "v", 1, "md", -1); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: put(): invalid negative metadata size: -1 (cache_path: " + TEST_DB + ")", e.what()); } // take() with empty key try { c.take("", out_val); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("x", out_val); EXPECT_EQ("PersistentStringCache: take(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // invalidate() with empty key try { c.invalidate(""); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: invalidate(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // touch() with empty key try { c.touch(""); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: touch(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // get_metadata() with empty key try { string md; c.get_metadata("", md); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: get_metadata(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // put_metadata() with empty key try { c.put_metadata("", "a"); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: put_metadata(): key must be non-empty (cache_path: " + TEST_DB + ")", e.what()); } // put_metadata() with nullptr try { c.put_metadata("1", nullptr, 1); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ( "PersistentStringCache: put_metadata(): metadata must not be nullptr (cache_path: " + TEST_DB + ")", e.what()); } // put_metadata() with negative metadata size try { c.put_metadata("1", "a", -1); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ("PersistentStringCache: put_metadata(): invalid negative size: -1 (cache_path: " + TEST_DB + ")", e.what()); } // put_metadata() with excessive size try { c.invalidate(); EXPECT_TRUE(c.put("1", "")); string meta(c.max_size_in_bytes() - 1, 'a'); EXPECT_TRUE(c.put_metadata("1", meta)); // OK, right at the limit meta += 'a'; c.put_metadata("1", meta); // One byte too large. FAIL(); } catch (logic_error const& e) { EXPECT_EQ(string("PersistentStringCache: put_metadata(): cannot add 1024-byte metadata: ") + "record size (1025) exceeds maximum cache size of 1024 (cache_path: " + TEST_DB + ")", e.what()); } } // touch() and put() with expiry time on lru_only DB. { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_only); auto expiry_time = chrono::system_clock::now() + chrono::milliseconds(1000); try { EXPECT_TRUE(c.put("x", "x")); c.touch("x", expiry_time); FAIL(); } catch (logic_error const& e) { string msg = string("PersistentStringCache: touch(): policy is lru_only, but expiry_time (") + to_string((chrono::duration_cast(expiry_time.time_since_epoch())).count()) + ") is not infinite (cache_path: " + TEST_DB + ")"; EXPECT_EQ(msg, e.what()); } try { c.put("y", "y", expiry_time); FAIL(); } catch (logic_error const& e) { string msg = string("PersistentStringCache: put(): policy is lru_only, but expiry_time (") + to_string((chrono::duration_cast(expiry_time.time_since_epoch())).count()) + ") is not infinite (cache_path: " + TEST_DB + ")"; EXPECT_EQ(msg, e.what()); } { bool handler_called = false; auto handler = [&](string const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; throw 42; }; // For coverage: check that throwing handlers don't do damage, // and that we can cancel handlers. c.set_handler(AllCacheEvents, handler); string val; EXPECT_FALSE(c.get("no_such_key", val)); EXPECT_TRUE(handler_called); c.set_handler(AllCacheEvents, nullptr); EXPECT_NO_THROW(c.invalidate("no_such_key")); try { CacheEvent events = CacheEvent(0); c.set_handler(events, handler); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ( "PersistentStringCache: set_handler(): invalid events (0): value must be in the " "range [1..127] (cache_path: " + TEST_DB + ")", e.what()); } try { CacheEvent events = CacheEvent::END_; c.set_handler(events, handler); FAIL(); } catch (invalid_argument const& e) { EXPECT_EQ( "PersistentStringCache: set_handler(): invalid events (128): value must be in the " "range [1..127] (cache_path: " + TEST_DB + ")", e.what()); } } // Tests that follow expect non-empty DB. ASSERT_NE(0, c.size()); } unique_ptr db; leveldb::WriteOptions write_options; auto open_db = [&]() { leveldb::Options options; leveldb::DB* p; auto s = leveldb::DB::Open(options, TEST_DB, &p); ASSERT_TRUE(s.ok()); db.reset(p); }; { unlink_db(TEST_DB); { PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_only); EXPECT_TRUE(c.put("y", "y")); } { // Write a garbage value into the version. open_db(); string val = "nan"; auto s = db->Put(write_options, "YSCHEMA_VERSION", val); ASSERT_TRUE(s.ok()); db.reset(nullptr); } { try { PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_only); FAIL(); } catch (system_error const& e) { EXPECT_EQ("PersistentStringCache: check_version(): bad version: \"nan\" (cache_path: " + TEST_DB + "): Unknown error 666", e.what()); } } { // Write a version mismatch. open_db(); string val = "0"; auto s = db->Put(write_options, "YSCHEMA_VERSION", val); ASSERT_TRUE(s.ok()); db.reset(nullptr); } { // Must succeed and will silently wipe the DB. PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_only); EXPECT_EQ(0, c.size()); EXPECT_TRUE(c.put("y", "y")); } { // Write a version mismatch. open_db(); string val = "0"; auto s = db->Put(write_options, "YSCHEMA_VERSION", val); ASSERT_TRUE(s.ok()); db.reset(nullptr); } { // Same as previous test, but using the other constructor. // Must succeed and will silently wipe the DB. PersistentStringCacheImpl c(TEST_DB); EXPECT_EQ(0, c.size()); } } } TEST(PersistentStringCacheImpl, insert_small) { unlink_db(TEST_DB); const int num = 99; PersistentStringCacheImpl c(TEST_DB, 1024 * 1024 * 1024, CacheDiscardPolicy::lru_ttl); // Insert num records, each with a 10 KB value string b(10 * 1024, 'b'); for (int i = 0; i < num; ++i) { string key = to_string(i); c.put(key, b); } EXPECT_EQ(num, c.size()); } TEST(PersistentStringCacheImpl, trim_small) { // No unlink here, we trim the result of the previous test PersistentStringCacheImpl c(TEST_DB, 1024 * 1024 * 1024, CacheDiscardPolicy::lru_ttl); c.trim_to(11 * 1024); EXPECT_LE(c.size(), 1); // trim_to() may remove more than asked for. if (c.size() == 1) { EXPECT_EQ(10 * 1024 + 2, c.size_in_bytes()); // Last record inserted had key "99" (2 chars long) } } TEST(PersistentStringCacheImpl, insert_large) { unlink_db(TEST_DB); const int num = 99; PersistentStringCacheImpl c(TEST_DB, 100 * 1024 * 1024, CacheDiscardPolicy::lru_ttl); // Insert num records, each with a 1 MB value string b(1024 * 1024, 'b'); for (int i = 0; i < num; ++i) { string key = to_string(i); c.put(key, b); } EXPECT_EQ(num, c.size()); } TEST(PersistentStringCacheImpl, trim_large) { // No unlink here, we trim the result of the previous test PersistentStringCacheImpl c(TEST_DB); c.trim_to(10 * 1024 * 1024); EXPECT_LE(c.size(), 10); c.trim_to(0); EXPECT_EQ(0, c.size()); } TEST(PersistentStringCacheImpl, resize) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 3 * 1024, CacheDiscardPolicy::lru_ttl); EXPECT_EQ(3 * 1024, c.max_size_in_bytes()); string b(1023, 'b'); c.put("a", b); c.put("b", b); this_thread::sleep_for(chrono::milliseconds(20)); c.put("c", b); c.resize(6 * 1024); EXPECT_EQ(6 * 1024, c.max_size_in_bytes()); EXPECT_EQ(3, c.size()); EXPECT_EQ(3 * 1024, c.size_in_bytes()); EXPECT_TRUE(c.contains_key("a")); EXPECT_TRUE(c.contains_key("b")); EXPECT_TRUE(c.contains_key("c")); c.resize(1 * 1024); EXPECT_EQ(1 * 1024, c.max_size_in_bytes()); EXPECT_EQ(1 * 1024, c.size_in_bytes()); EXPECT_EQ(1, c.size()); EXPECT_FALSE(c.contains_key("a")); EXPECT_FALSE(c.contains_key("b")); EXPECT_TRUE(c.contains_key("c")); } TEST(PersistentStringCacheImpl, insert_when_full) { unlink_db(TEST_DB); const int num = 50; PersistentStringCacheImpl c(TEST_DB, 10 * 1024, CacheDiscardPolicy::lru_ttl); // Enough for 9 records EXPECT_EQ(0, c.size()); EXPECT_EQ(10 * 1024, c.max_size_in_bytes()); // Insert num records, each a little over 1 KB in size string b(1024, 'b'); for (int i = 0; i < num; ++i) { c.put(to_string(i), b); } // At most nine records because key length pushes size over the 1024 KB size. // Depending on how the access time stamps fall out, we may actually end up // with one record (if the preceding 9 records were inserted in the same millisecond). EXPECT_GE(c.size(), 1); EXPECT_LE(c.size(), 9); } TEST(PersistentStringCacheImpl, invalidate) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1 * 1024 * 1024 * 1024, CacheDiscardPolicy::lru_only); c.invalidate(); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); // Insert num records, each a little over 1 KB in size const int num = 10768; string b(1024, 'b'); for (int i = 0; i < num; ++i) { c.put(to_string(i), b); } c.invalidate(); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); // Vector version. vector keys; for (int i = 0; i < num; ++i) { keys.push_back(to_string(i)); c.put(to_string(i), b); } c.invalidate(keys); EXPECT_EQ(0, c.size()); EXPECT_EQ(0, c.size_in_bytes()); // For coverage mainly, and to verify that compact() indeed compacts the DB. c.invalidate(); c.compact(); EXPECT_LT(c.disk_size_in_bytes(), 1000); } TEST(PersistentStringCacheImpl, stats) { PersistentCacheStats original_stats; { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 128, CacheDiscardPolicy::lru_only); auto s = c.stats(); auto hist = s.histogram(); for (unsigned i = 0; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Histogram must be empty } c.put("x", "y"); string val; EXPECT_TRUE(c.get("x", val)); EXPECT_EQ("y", val); s = c.stats(); EXPECT_EQ(1, s.size()); EXPECT_EQ(2, s.size_in_bytes()); EXPECT_EQ(128, s.max_size_in_bytes()); EXPECT_EQ(1, s.hits()); c.clear_stats(); s = c.stats(); EXPECT_EQ(1, s.size()); EXPECT_EQ(2, s.size_in_bytes()); EXPECT_EQ(128, s.max_size_in_bytes()); EXPECT_EQ(0, s.hits()); EXPECT_EQ(1u, s.histogram()[0]); c.put("x", "y"); // Value was already there s = c.stats(); hist = s.histogram(); EXPECT_EQ(1u, hist[0]); for (unsigned i = 1; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); } c.put("y", ""); // New value s = c.stats(); hist = s.histogram(); EXPECT_EQ(2u, hist[0]); for (unsigned i = 1; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); } c.put("y", "ab"); // Replace value with larger one in same bin. s = c.stats(); hist = s.histogram(); EXPECT_EQ(2u, hist[0]); // Bin count must still be the same. for (unsigned i = 1; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); } c.put("y", string(9, 'y')); // Replace value with larger one in next bin. s = c.stats(); hist = s.histogram(); EXPECT_EQ(1u, hist[0]); EXPECT_EQ(1u, hist[1]); // Value must have moved to new bin. for (unsigned i = 2; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } c.put_metadata("y", string(1, 'm')); // Add small metadata, value stays in same bin. s = c.stats(); hist = s.histogram(); EXPECT_EQ(1u, hist[0]); EXPECT_EQ(1u, hist[1]); // Value must have moved to new bin. for (unsigned i = 2; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } c.put_metadata("y", string(10, 'm')); // Add larger metadata, value moves to next bin. s = c.stats(); hist = s.histogram(); EXPECT_EQ(1u, hist[0]); EXPECT_EQ(0u, hist[1]); EXPECT_EQ(1u, hist[2]); for (unsigned i = 3; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } c.put_metadata("y", string(1, 'm')); // Shrink metadata, value moves to previous bin. s = c.stats(); hist = s.histogram(); EXPECT_EQ(1u, hist[0]); EXPECT_EQ(1u, hist[1]); for (unsigned i = 2; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]) << i; // Other bins must still be empty. } c.put("new key", string(1, 'k')); s = c.stats(); EXPECT_EQ(3, s.size()); c.get("new key", val); s = c.stats(); EXPECT_EQ(1, s.hits()); c.invalidate(); s = c.stats(); hist = s.histogram(); for (unsigned i = 0; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Histogram must have been emptied. } EXPECT_EQ(0, s.hits()); // invalidate() also clears the ephemeral stats. c.put("1", string(1, 'k')); // First bin c.put("2", string(10, 'k')); // Second bin c.put("3", string(20, 'k')); // Third bin c.put("4", string(30, 'k')); // Fourth bin c.invalidate({"2", "3"}); s = c.stats(); hist = s.histogram(); EXPECT_EQ(1u, hist[0]); EXPECT_EQ(0u, hist[1]); EXPECT_EQ(0u, hist[2]); EXPECT_EQ(1u, hist[3]); for (unsigned i = 4; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } c.invalidate("1"); // Invalidate specific entry s = c.stats(); hist = s.histogram(); EXPECT_EQ(0u, hist[0]); EXPECT_EQ(0u, hist[1]); EXPECT_EQ(0u, hist[2]); EXPECT_EQ(1u, hist[3]); for (unsigned i = 4; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } // Rather than testing all 74 bins, we test a few critical ones. // If they are right, so will be the others, seeing that they // are generated. auto bounds = PersistentCacheStats::histogram_bounds(); typedef pair P; EXPECT_EQ(P(1, 9), bounds[0]); EXPECT_EQ(P(10, 19), bounds[1]); EXPECT_EQ(P(20, 29), bounds[2]); EXPECT_EQ(P(90, 99), bounds[9]); EXPECT_EQ(P(100, 199), bounds[10]); EXPECT_EQ(P(900, 999), bounds[18]); EXPECT_EQ(P(900000000, 999999999), bounds[72]); EXPECT_EQ(P(1000000000, numeric_limits::max()), bounds[73]); // Generate a few hits and misses. for (auto i = 0; i < 10; ++i) { c.get(to_string(i), val); } s = c.stats(); original_stats = s; EXPECT_EQ(1, original_stats.hits()); EXPECT_EQ(9, original_stats.misses()); EXPECT_EQ(0, original_stats.hits_since_last_miss()); EXPECT_EQ(5, original_stats.misses_since_last_hit()); EXPECT_EQ(1, original_stats.longest_hit_run()); EXPECT_EQ(5, original_stats.longest_miss_run()); EXPECT_EQ(0, original_stats.ttl_evictions()); EXPECT_EQ(0, original_stats.lru_evictions()); EXPECT_NE(chrono::system_clock::time_point(), original_stats.most_recent_hit_time()); EXPECT_NE(chrono::system_clock::time_point(), original_stats.most_recent_miss_time()); EXPECT_NE(chrono::system_clock::time_point(), original_stats.longest_hit_run_time()); EXPECT_NE(chrono::system_clock::time_point(), original_stats.longest_miss_run_time()); } { using namespace std::chrono; // Re-open previous cache. PersistentStringCacheImpl c(TEST_DB); auto s = c.stats(); // size and size_in_bytes must be the same. EXPECT_EQ(1, s.size()); EXPECT_EQ(31, s.size_in_bytes()); // Histogram must be re-established when opened. auto hist = s.histogram(); EXPECT_EQ(0u, hist[0]); EXPECT_EQ(0u, hist[1]); EXPECT_EQ(0u, hist[2]); EXPECT_EQ(1u, hist[3]); for (unsigned i = 4; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } // Ephemeral counters must still be intact. EXPECT_EQ(original_stats.hits(), s.hits()); EXPECT_EQ(original_stats.misses(), s.misses()); EXPECT_EQ(original_stats.hits_since_last_miss(), s.hits_since_last_miss()); EXPECT_EQ(original_stats.misses_since_last_hit(), s.misses_since_last_hit()); EXPECT_EQ(original_stats.longest_hit_run(), s.longest_hit_run()); EXPECT_EQ(original_stats.longest_miss_run(), s.longest_miss_run()); EXPECT_EQ(original_stats.hit_runs(), s.hit_runs()); EXPECT_EQ(original_stats.miss_runs(), s.miss_runs()); EXPECT_EQ(original_stats.avg_hit_run_length(), s.avg_hit_run_length()); EXPECT_EQ(original_stats.avg_miss_run_length(), s.avg_miss_run_length()); EXPECT_EQ(original_stats.ttl_evictions(), s.ttl_evictions()); EXPECT_EQ(original_stats.lru_evictions(), s.lru_evictions()); // Time stamps must still be intact. auto orig_t = duration_cast(original_stats.most_recent_hit_time().time_since_epoch()).count(); auto s_t = duration_cast(s.most_recent_hit_time().time_since_epoch()).count(); EXPECT_EQ(orig_t, s_t); orig_t = duration_cast(original_stats.most_recent_miss_time().time_since_epoch()).count(); s_t = duration_cast(s.most_recent_miss_time().time_since_epoch()).count(); EXPECT_EQ(orig_t, s_t); orig_t = duration_cast(original_stats.longest_hit_run_time().time_since_epoch()).count(); s_t = duration_cast(s.longest_hit_run_time().time_since_epoch()).count(); EXPECT_EQ(orig_t, s_t); orig_t = duration_cast(original_stats.longest_miss_run_time().time_since_epoch()).count(); s_t = duration_cast(s.longest_miss_run_time().time_since_epoch()).count(); EXPECT_EQ(orig_t, s_t); } { // Simulate crash by clearing the dirty flag. unique_ptr db; leveldb::Options options; leveldb::DB* p; auto s = leveldb::DB::Open(options, TEST_DB, &p); ASSERT_TRUE(s.ok()); db.reset(p); leveldb::WriteOptions write_options; string val = "1"; s = db->Put(write_options, "!DIRTY", val); ASSERT_TRUE(s.ok()); }; { using namespace std::chrono; // Re-open previous (now dirty) cache. PersistentStringCacheImpl c(TEST_DB); auto s = c.stats(); // size and size_in_bytes must be the same. EXPECT_EQ(1, s.size()); EXPECT_EQ(31, s.size_in_bytes()); // Histogram must be re-established when opened. auto hist = s.histogram(); EXPECT_EQ(0u, hist[0]); EXPECT_EQ(0u, hist[1]); EXPECT_EQ(0u, hist[2]); EXPECT_EQ(1u, hist[3]); for (unsigned i = 4; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } // Ephemeral counters must all be zero. EXPECT_EQ(0, s.hits()); EXPECT_EQ(0, s.misses()); EXPECT_EQ(0, s.hits_since_last_miss()); EXPECT_EQ(0, s.misses_since_last_hit()); EXPECT_EQ(0, s.longest_hit_run()); EXPECT_EQ(0, s.longest_miss_run()); EXPECT_EQ(0, s.hit_runs()); EXPECT_EQ(0, s.miss_runs()); EXPECT_EQ(0.0, s.avg_hit_run_length()); EXPECT_EQ(0.0, s.avg_miss_run_length()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); // Time stamps must all be at the epoch. EXPECT_EQ(system_clock::time_point(), s.most_recent_hit_time()); EXPECT_EQ(system_clock::time_point(), s.most_recent_miss_time()); EXPECT_EQ(system_clock::time_point(), s.longest_hit_run_time()); EXPECT_EQ(system_clock::time_point(), s.longest_miss_run_time()); } } TEST(PersistentStringCacheImpl, run_length_stats) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 128, CacheDiscardPolicy::lru_only); c.put("x", "x"); // Just so we have something we can generate a hit on. auto s = c.stats(); EXPECT_EQ(0, s.hits()); EXPECT_EQ(0, s.misses()); EXPECT_EQ(0, s.hit_runs()); EXPECT_EQ(0, s.miss_runs()); EXPECT_EQ(0.0, s.avg_hit_run_length()); EXPECT_EQ(0.0, s.avg_miss_run_length()); string val; c.get("not there", val); // Miss s = c.stats(); EXPECT_EQ(0, s.hits()); EXPECT_EQ(1, s.misses()); EXPECT_EQ(0, s.hit_runs()); EXPECT_EQ(1, s.miss_runs()); // Just started a miss run EXPECT_EQ(0.0, s.avg_hit_run_length()); EXPECT_EQ(1.0, s.avg_miss_run_length()); c.get("not there", val); // Another miss s = c.stats(); EXPECT_EQ(0, s.hits()); EXPECT_EQ(2, s.misses()); EXPECT_EQ(0, s.hit_runs()); EXPECT_EQ(1, s.miss_runs()); // Still in the same run EXPECT_EQ(0.0, s.avg_hit_run_length()); EXPECT_EQ(2.0, s.avg_miss_run_length()); c.get("x", val); // Hit s = c.stats(); EXPECT_EQ(1, s.hits()); EXPECT_EQ(2, s.misses()); EXPECT_EQ(1, s.hit_runs()); // Just started a hit run EXPECT_EQ(1, s.miss_runs()); EXPECT_EQ(1.0, s.avg_hit_run_length()); EXPECT_EQ(2.0, s.avg_miss_run_length()); // Last run was two misses c.get("not there", val); // Another miss s = c.stats(); EXPECT_EQ(1, s.hits()); EXPECT_EQ(3, s.misses()); EXPECT_EQ(1, s.hit_runs()); EXPECT_EQ(2, s.miss_runs()); // Just started a new miss run EXPECT_EQ(1.0, s.avg_hit_run_length()); EXPECT_EQ(1.5, s.avg_miss_run_length()); // Last run was two misses for (int i = 0; i < 3; ++i) // Three more misses { c.get("not there", val); } s = c.stats(); EXPECT_EQ(1, s.hits()); EXPECT_EQ(6, s.misses()); EXPECT_EQ(1, s.hit_runs()); EXPECT_EQ(2, s.miss_runs()); // Just started a new miss run EXPECT_EQ(1.0, s.avg_hit_run_length()); EXPECT_EQ(3.0, s.avg_miss_run_length()); c.get("x", val); // Hit s = c.stats(); EXPECT_EQ(2, s.hits()); EXPECT_EQ(6, s.misses()); EXPECT_EQ(2, s.hit_runs()); EXPECT_EQ(2, s.miss_runs()); EXPECT_EQ(1.0, s.avg_hit_run_length()); EXPECT_EQ(3.0, s.avg_miss_run_length()); // 6 missing in 2 runs. } TEST(PersistentStringCacheImpl, run_length_stats_restart) { // Don't unlink the DB here, we need the contents from the previous test. PersistentStringCacheImpl c(TEST_DB, 128, CacheDiscardPolicy::lru_only); string val; auto s = c.stats(); // Stats from pervious test must still be intact. EXPECT_EQ(2, s.hits()); EXPECT_EQ(6, s.misses()); EXPECT_EQ(2, s.hit_runs()); EXPECT_EQ(2, s.miss_runs()); EXPECT_EQ(1.0, s.avg_hit_run_length()); EXPECT_EQ(3.0, s.avg_miss_run_length()); for (int i = 0; i < 18; ++i) { c.get("x", val); // Hit } // Finish hit run, and add six more misses. for (int i = 0; i < 6; ++i) { c.get("not there", val); } s = c.stats(); EXPECT_EQ(20, s.hits()); EXPECT_EQ(12, s.misses()); EXPECT_EQ(2, s.hit_runs()); EXPECT_EQ(3, s.miss_runs()); EXPECT_EQ(10.0, s.avg_hit_run_length()); EXPECT_EQ(4.0, s.avg_miss_run_length()); } TEST(PersistentStringCacheImpl, event_handlers) { unlink_db(TEST_DB); PersistentStringCacheImpl c(TEST_DB, 1024, CacheDiscardPolicy::lru_ttl); // Copied from PersistentStringCache.h because the definition is private there. static constexpr unsigned END_ = 7; struct EventRecord { CacheEvent ev; PersistentCacheStats stats; }; // A map for each event type. The inner map records the key and event details. map> event_maps; for (unsigned i = 0; i < END_; ++i) { auto current_event = static_cast(1 << i); auto get_handler = [&c, &event_maps, current_event](string const& key, CacheEvent ev, PersistentCacheStats const& stats) { event_maps[current_event][key] = {ev, stats}; }; c.set_handler(current_event, get_handler); } string val; EventRecord er; auto map = &event_maps[CacheEvent::put]; // Check Put events. c.put("1", "x"); ASSERT_EQ(1u, map->size()); er = (*map)["1"]; EXPECT_EQ(CacheEvent::put, er.ev); EXPECT_EQ(1, er.stats.size()); EXPECT_EQ(2, er.stats.size_in_bytes()); this_thread::sleep_for(chrono::milliseconds(5)); // Make sure we have different time stamps. c.put("2", "x"); ASSERT_EQ(2u, map->size()); er = (*map)["2"]; EXPECT_EQ(CacheEvent::put, er.ev); EXPECT_EQ(2, er.stats.size()); EXPECT_EQ(4, er.stats.size_in_bytes()); this_thread::sleep_for(chrono::milliseconds(5)); c.put("3", "x"); ASSERT_EQ(3u, map->size()); er = (*map)["3"]; EXPECT_EQ(CacheEvent::put, er.ev); EXPECT_EQ(3, er.stats.size()); EXPECT_EQ(6, er.stats.size_in_bytes()); this_thread::sleep_for(chrono::milliseconds(5)); c.put("4", "x"); ASSERT_EQ(4u, map->size()); er = (*map)["4"]; EXPECT_EQ(CacheEvent::put, er.ev); EXPECT_EQ(4, er.stats.size()); EXPECT_EQ(8, er.stats.size_in_bytes()); // Check Get event. this_thread::sleep_for(chrono::milliseconds(5)); c.get("3", val); map = &event_maps[CacheEvent::get]; ASSERT_EQ(1u, map->size()); er = (*map)["3"]; EXPECT_EQ(CacheEvent::get, er.ev); EXPECT_EQ(4, er.stats.size()); EXPECT_EQ(8, er.stats.size_in_bytes()); map->clear(); // Check invalidate and take. map = &event_maps[CacheEvent::invalidate]; c.invalidate("1"); ASSERT_EQ(1u, map->size()); er = (*map)["1"]; EXPECT_EQ(CacheEvent::invalidate, er.ev); EXPECT_EQ(3, er.stats.size()); EXPECT_EQ(6, er.stats.size_in_bytes()); map->clear(); c.take("2", val); map = &event_maps[CacheEvent::get]; ASSERT_EQ(1u, map->size()); er = (*map)["2"]; EXPECT_EQ(CacheEvent::get, er.ev); EXPECT_EQ(2, er.stats.size()); EXPECT_EQ(4, er.stats.size_in_bytes()); map->clear(); map = &event_maps[CacheEvent::invalidate]; ASSERT_EQ(1u, map->size()); er = (*map)["2"]; EXPECT_EQ(CacheEvent::invalidate, er.ev); EXPECT_EQ(2, er.stats.size()); EXPECT_EQ(4, er.stats.size_in_bytes()); map->clear(); c.invalidate(); ASSERT_EQ(2u, map->size()); er = (*map)["4"]; EXPECT_EQ(CacheEvent::invalidate, er.ev); EXPECT_EQ(1, er.stats.size()); EXPECT_EQ(2, er.stats.size_in_bytes()); // 3 was accessed last, so it must be removed last. er = (*map)["3"]; EXPECT_EQ(CacheEvent::invalidate, er.ev); EXPECT_EQ(0, er.stats.size()); EXPECT_EQ(0, er.stats.size_in_bytes()); map->clear(); // Check touch c.put("1", "1"); map = &event_maps[CacheEvent::touch]; c.touch("1"); ASSERT_EQ(1u, map->size()); er = (*map)["1"]; EXPECT_EQ(CacheEvent::touch, er.ev); EXPECT_EQ(1, er.stats.size()); EXPECT_EQ(2, er.stats.size_in_bytes()); c.invalidate(); // Check misses string bad_key = "no_such_key"; map = &event_maps[CacheEvent::miss]; c.get(bad_key, val); ASSERT_EQ(1u, map->size()); er = (*map)[bad_key]; EXPECT_EQ(CacheEvent::miss, er.ev); EXPECT_EQ(0, er.stats.size()); EXPECT_EQ(0, er.stats.size_in_bytes()); map->clear(); c.invalidate(); auto later = chrono::system_clock::now() + chrono::milliseconds(50); c.put(bad_key, "", later); while (chrono::system_clock::now() <= later) { this_thread::sleep_for(chrono::milliseconds(5)); } c.get(bad_key, val); // Already expired, so we must get a miss. ASSERT_EQ(1u, map->size()); er = (*map)[bad_key]; EXPECT_EQ(CacheEvent::miss, er.ev); EXPECT_EQ(1, er.stats.size()); EXPECT_EQ(int(bad_key.size()), er.stats.size_in_bytes()); c.invalidate(); map->clear(); event_maps[CacheEvent::invalidate].clear(); event_maps[CacheEvent::miss].clear(); later = chrono::system_clock::now() + chrono::milliseconds(50); c.put(bad_key, "", later); this_thread::sleep_for(chrono::milliseconds(60)); c.invalidate(bad_key); // Already expired, so we must get an invalidate, but not a miss. map = &event_maps[CacheEvent::miss]; ASSERT_EQ(0u, map->size()); map = &event_maps[CacheEvent::invalidate]; ASSERT_EQ(1u, map->size()); er = (*map)[bad_key]; EXPECT_EQ(CacheEvent::invalidate, er.ev); EXPECT_EQ(0, er.stats.size()); EXPECT_EQ(0, er.stats.size_in_bytes()); c.invalidate(); map->clear(); // Check evict_ttl later = chrono::system_clock::now() + chrono::milliseconds(100); c.put("1", "", later); this_thread::sleep_for(chrono::milliseconds(10)); later = chrono::system_clock::now() + chrono::milliseconds(100); c.put("2", "", later); while (chrono::system_clock::now() <= later) { this_thread::sleep_for(chrono::milliseconds(5)); } // Both entries have expired now. c.trim_to(1); // Both entries have expired. Even though we asked for a trim_to(1), // both entries will be deleted as part of the trim_to(). map = &event_maps[CacheEvent::evict_ttl]; ASSERT_EQ(2u, map->size()); er = (*map)["1"]; EXPECT_EQ(CacheEvent::evict_ttl, er.ev); // Entry "1" expired first so, when it is deleted, entry "2" is still around. EXPECT_EQ(1, er.stats.size()); EXPECT_EQ(1, er.stats.size_in_bytes()); er = (*map)["2"]; EXPECT_EQ(CacheEvent::evict_ttl, er.ev); // Entry "2" expired second. EXPECT_EQ(0, er.stats.size()); EXPECT_EQ(0, er.stats.size_in_bytes()); // Check evict_lru c.put("1", ""); c.put("2", ""); c.trim_to(0); map = &event_maps[CacheEvent::evict_lru]; ASSERT_EQ(2u, map->size()); er = (*map)["1"]; EXPECT_EQ(CacheEvent::evict_lru, er.ev); // Entry "1" is oldest, so gets evicted first. EXPECT_EQ(1, er.stats.size()); EXPECT_EQ(1, er.stats.size_in_bytes()); er = (*map)["2"]; EXPECT_EQ(CacheEvent::evict_lru, er.ev); // Entry "2" is youngest, so it gets deleted last. EXPECT_EQ(0, er.stats.size()); EXPECT_EQ(0, er.stats.size_in_bytes()); } persistent-cache-cpp-1.0.5/tests/core/persistent_cache/000077500000000000000000000000001414021052300231435ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/core/persistent_cache/CMakeLists.txt000066400000000000000000000005301414021052300257010ustar00rootroot00000000000000add_executable(persistent_cache_test persistent_cache_test.cpp) target_link_libraries(persistent_cache_test ${TESTLIBS}) add_definitions(-DTEST_DIR="${CMAKE_CURRENT_BINARY_DIR}") add_test(persistent_cache persistent_cache_test) set(TARGETS ${TARGETS} persistent_cache_test) set(UNIT_TEST_TARGETS ${UNIT_TEST_TARGETS} ${TARGETS} PARENT_SCOPE) persistent-cache-cpp-1.0.5/tests/core/persistent_cache/persistent_cache_test.cpp000066400000000000000000001132721414021052300302370ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wctor-dtor-privacy" #include #pragma GCC diagnostic pop using namespace std; using namespace core; string const test_db = TEST_DIR "/db"; // Removes the contents of db_dir, but not db_dir itself. void unlink_db(string const& db_dir) { namespace fs = boost::filesystem; try { for (fs::directory_iterator end, it(db_dir); it != end; ++it) { remove_all(it->path()); } } catch (...) { } } namespace core { template <> string CacheCodec::encode(int const& value) { return to_string(value); } template <> int CacheCodec::decode(string const& s) { return stoi(s); } template <> string CacheCodec::encode(double const& value) { return to_string(value); } template <> double CacheCodec::decode(string const& s) { return stod(s); } template <> string CacheCodec::encode(char const& value) { return string(1, value); } template <> char CacheCodec::decode(string const& s) { return s.empty() ? '\0' : s[0]; } } // namespace core TEST(PersistentCache, IDCCache) { unlink_db(test_db); using IDCCache = PersistentCache; { // Constructor and move constructor. auto c = IDCCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); IDCCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = IDCCache::open(test_db); auto c2 = IDCCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = IDCCache::open(test_db); auto val = c->get(1); EXPECT_FALSE(val); auto data = c->get_data(1); EXPECT_FALSE(data); auto metadata = c->get_metadata(1); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key(1)); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take(42); EXPECT_FALSE(val); data = c->take_data(42); EXPECT_FALSE(data); EXPECT_TRUE(c->put(1, 2.0)); val = c->get(1); EXPECT_EQ(2.0, *val); data = c->get_data(1); EXPECT_EQ(2.0, data->value); EXPECT_EQ('\0', data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata(1); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate(1)); EXPECT_TRUE(c->put(1, 2.0)); data = c->take_data(1); EXPECT_TRUE(bool(data)); EXPECT_EQ(2.0, data->value); EXPECT_EQ('\0', data->metadata); EXPECT_TRUE(c->put(2, 3, '4')); data = c->take_data(2); EXPECT_TRUE(bool(data)); EXPECT_EQ(3, data->value); EXPECT_EQ('4', data->metadata); EXPECT_TRUE(c->put(1, 2, '3')); EXPECT_TRUE(c->put_metadata(1, '3')); data = c->take_data(1); EXPECT_EQ(2, data->value); EXPECT_EQ('3', data->metadata); val = c->take(42); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate(42)); EXPECT_FALSE(c->touch(42)); c->invalidate(); c->compact(); EXPECT_TRUE(c->put(1, 0)); EXPECT_TRUE(c->put(2, 0)); c->invalidate({1, 2}); EXPECT_FALSE(c->contains_key(1)); EXPECT_FALSE(c->contains_key(2)); EXPECT_TRUE(c->put(3, 0)); EXPECT_TRUE(c->put(4, 0)); vector keys = {3, 4}; c->invalidate(keys); EXPECT_FALSE(c->contains_key(3)); EXPECT_FALSE(c->contains_key(4)); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); // Handlers bool handler_called; auto handler = [&](int const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put(1, 1)); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put(2, 2)); EXPECT_TRUE(handler_called); // Event operators typedef std::underlying_type::type EventValue; EXPECT_EQ(0x7fu, EventValue(AllCacheEvents)); EXPECT_EQ(0x7eu, EventValue(~CacheEvent::get)); EXPECT_EQ(0x3u, EventValue(CacheEvent::get | CacheEvent::put)); EXPECT_EQ(0x2u, EventValue(AllCacheEvents & CacheEvent::put)); CacheEvent v = CacheEvent::get | CacheEvent::put; v |= CacheEvent::invalidate; EXPECT_EQ(0x7u, EventValue(v)); v &= ~CacheEvent::get; EXPECT_EQ(0x6u, EventValue(v)); // Loader methods bool loader_called; auto loader = [&](int const& key, IDCCache& c) { loader_called = true; if (key != 99) { EXPECT_TRUE(c.put(key, 99)); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put(3, loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data(4, loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data(99, loader))); } } // Test below go through the seven specializations for the different // compbinations of custom type and string. // K = string TEST(PersistentCache, SDCCache) { unlink_db(test_db); using SDCCache = PersistentCache; { // Constructor and move constructor. auto c = SDCCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); SDCCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = SDCCache::open(test_db); auto c2 = SDCCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = SDCCache::open(test_db); auto val = c->get("1"); EXPECT_FALSE(val); auto data = c->get_data("1"); EXPECT_FALSE(data); auto metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key("1")); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take("42"); EXPECT_FALSE(val); data = c->take_data("42"); EXPECT_FALSE(data); EXPECT_TRUE(c->put("1", 2.0)); val = c->get("1"); EXPECT_EQ(2.0, *val); data = c->get_data("1"); EXPECT_EQ(2.0, data->value); EXPECT_EQ('\0', data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate("1")); EXPECT_TRUE(c->put("1", 2.0)); data = c->take_data("1"); EXPECT_TRUE(bool(data)); EXPECT_EQ(2.0, data->value); EXPECT_EQ('\0', data->metadata); EXPECT_TRUE(c->put("2", 3, '4')); data = c->take_data("2"); EXPECT_TRUE(bool(data)); EXPECT_EQ(3, data->value); EXPECT_EQ('4', data->metadata); EXPECT_TRUE(c->put("1", 2, '3')); EXPECT_TRUE(c->put_metadata("1", '3')); data = c->take_data("1"); EXPECT_EQ(2, data->value); EXPECT_EQ('3', data->metadata); val = c->take("42"); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate("42")); EXPECT_FALSE(c->touch("42")); c->invalidate(); c->compact(); EXPECT_TRUE(c->put("1", 0)); EXPECT_TRUE(c->put("2", 0)); c->invalidate({"1", "2"}); EXPECT_FALSE(c->contains_key("1")); EXPECT_FALSE(c->contains_key("2")); EXPECT_TRUE(c->put("3", 0)); EXPECT_TRUE(c->put("3", 0)); vector keys = {{"3"}, {"4"}}; c->invalidate(keys); EXPECT_FALSE(c->contains_key("3")); EXPECT_FALSE(c->contains_key("3")); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](string const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put("1", 1)); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put("2", 2)); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](string const& key, SDCCache& c) { loader_called = true; if (key != "99") { EXPECT_TRUE(c.put(key, 99)); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put("3", loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data("4", loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data("99", loader))); } } // V = string TEST(PersistentCache, ISCCache) { unlink_db(test_db); using ISCCache = PersistentCache; { // Constructor and move constructor. auto c = ISCCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); ISCCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = ISCCache::open(test_db); auto c2 = ISCCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = ISCCache::open(test_db); auto val = c->get(1); EXPECT_FALSE(val); auto data = c->get_data(1); EXPECT_FALSE(data); auto metadata = c->get_metadata(1); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key(1)); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take(42); EXPECT_FALSE(val); data = c->take_data(42); EXPECT_FALSE(data); EXPECT_TRUE(c->put(1, "2.0")); val = c->get(1); EXPECT_EQ("2.0", *val); data = c->get_data(1); EXPECT_EQ("2.0", data->value); EXPECT_EQ('\0', data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata(1); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate(1)); EXPECT_TRUE(c->put(1, "2.0")); data = c->take_data(1); EXPECT_TRUE(bool(data)); EXPECT_EQ("2.0", data->value); EXPECT_EQ('\0', data->metadata); EXPECT_TRUE(c->put(2, string("3"), '4')); data = c->take_data(2); EXPECT_TRUE(bool(data)); EXPECT_EQ("3", data->value); EXPECT_EQ('4', data->metadata); EXPECT_TRUE(c->put(1, string("2"), '3')); EXPECT_TRUE(c->put_metadata(1, '3')); data = c->take_data(1); EXPECT_EQ("2", data->value); EXPECT_EQ('3', data->metadata); val = c->take(42); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate(42)); EXPECT_FALSE(c->touch(42)); c->invalidate(); c->compact(); EXPECT_TRUE(c->put(1, string("0"))); EXPECT_TRUE(c->put(2, string("0"))); c->invalidate({1, 2}); EXPECT_FALSE(c->contains_key(1)); EXPECT_FALSE(c->contains_key(2)); EXPECT_TRUE(c->put(3, string("0"))); EXPECT_TRUE(c->put(4, string("0"))); vector keys = {3, 4}; c->invalidate(keys); EXPECT_FALSE(c->contains_key(3)); EXPECT_FALSE(c->contains_key(4)); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](int const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put(1, "1")); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put(2, "2")); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](int const& key, ISCCache& c) { loader_called = true; if (key != 99) { EXPECT_TRUE(c.put(key, "99")); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put(3, loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data(4, loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data(99, loader))); // Extra put() overloads c->invalidate(); string vbuf(20, 'v'); EXPECT_TRUE(c->put(1, vbuf.data(), vbuf.size())); val = c->get(1); EXPECT_EQ(vbuf, *val); c->invalidate(); EXPECT_TRUE(c->put(1, vbuf.data(), vbuf.size(), 'm')); data = c->get_data(1); EXPECT_EQ(vbuf, data->value); EXPECT_EQ('m', data->metadata); } } // M = string TEST(PersistentCache, IDSCache) { unlink_db(test_db); using IDSCache = PersistentCache; { // Constructor and move constructor. auto c = IDSCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); IDSCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = IDSCache::open(test_db); auto c2 = IDSCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = IDSCache::open(test_db); auto val = c->get(1); EXPECT_FALSE(val); auto data = c->get_data(1); EXPECT_FALSE(data); auto metadata = c->get_metadata(1); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key(1)); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take(42); EXPECT_FALSE(val); data = c->take_data(42); EXPECT_FALSE(data); EXPECT_TRUE(c->put(1, 2.0)); val = c->get(1); EXPECT_EQ(2.0, *val); data = c->get_data(1); EXPECT_EQ(2.0, data->value); EXPECT_EQ("\0", data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata(1); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate(1)); EXPECT_TRUE(c->put(1, 2.0)); data = c->take_data(1); EXPECT_TRUE(bool(data)); EXPECT_EQ(2.0, data->value); EXPECT_EQ("\0", data->metadata); EXPECT_TRUE(c->put(2, 3, "4")); data = c->take_data(2); EXPECT_TRUE(bool(data)); EXPECT_EQ(3, data->value); EXPECT_EQ("4", data->metadata); EXPECT_TRUE(c->put(1, 2, "3")); EXPECT_TRUE(c->put_metadata(1, "3")); data = c->take_data(1); EXPECT_EQ(2, data->value); EXPECT_EQ("3", data->metadata); val = c->take(42); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate(42)); EXPECT_FALSE(c->touch(42)); c->invalidate(); EXPECT_TRUE(c->put(1, 0)); EXPECT_TRUE(c->put(2, 0)); c->invalidate({1, 2}); EXPECT_FALSE(c->contains_key(1)); EXPECT_FALSE(c->contains_key(2)); EXPECT_TRUE(c->put(3, 0)); EXPECT_TRUE(c->put(4, 0)); vector keys = {3, 4}; c->invalidate(keys); EXPECT_FALSE(c->contains_key(3)); EXPECT_FALSE(c->contains_key(4)); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](int const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put(1, 1)); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put(2, 2)); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](int const& key, IDSCache& c) { loader_called = true; if (key != 99) { EXPECT_TRUE(c.put(key, 99)); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put(3, loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data(4, loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data(99, loader))); // Extra put() overload c->invalidate(); c->compact(); string mbuf(20, 'm'); EXPECT_TRUE(c->put(1, 2.0, mbuf.data(), mbuf.size())); data = c->get_data(1); EXPECT_EQ(2.0, data->value); EXPECT_EQ(mbuf, data->metadata); mbuf = string(10, 'x'); EXPECT_TRUE(c->put_metadata(1, mbuf.data(), mbuf.size())); data = c->get_data(1); EXPECT_EQ(mbuf, data->metadata); } } // K and V = string TEST(PersistentCache, SSCCache) { unlink_db(test_db); using SSCCache = PersistentCache; { // Constructor and move constructor. auto c = SSCCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); SSCCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = SSCCache::open(test_db); auto c2 = SSCCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = SSCCache::open(test_db); auto val = c->get("1"); EXPECT_FALSE(val); auto data = c->get_data("1"); EXPECT_FALSE(data); auto metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key("1")); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take("42"); EXPECT_FALSE(val); data = c->take_data("42"); EXPECT_FALSE(data); EXPECT_TRUE(c->put("1", "2.0")); val = c->get("1"); EXPECT_EQ("2.0", *val); data = c->get_data("1"); EXPECT_EQ("2.0", data->value); EXPECT_EQ('\0', data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate("1")); EXPECT_TRUE(c->put("1", "2.0")); data = c->take_data("1"); EXPECT_TRUE(bool(data)); EXPECT_EQ("2.0", data->value); EXPECT_EQ('\0', data->metadata); EXPECT_TRUE(c->put("2", string("3"), '4')); data = c->take_data("2"); EXPECT_TRUE(bool(data)); EXPECT_EQ("3", data->value); EXPECT_EQ('4', data->metadata); EXPECT_TRUE(c->put("1", string("2"), '3')); EXPECT_TRUE(c->put_metadata("1", '3')); data = c->take_data("1"); EXPECT_EQ("2", data->value); EXPECT_EQ('3', data->metadata); val = c->take("42"); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate("42")); EXPECT_FALSE(c->touch("42")); c->invalidate(); c->compact(); EXPECT_TRUE(c->put("1", "0")); EXPECT_TRUE(c->put("2", "0")); c->invalidate({"1", "2"}); EXPECT_FALSE(c->contains_key("1")); EXPECT_FALSE(c->contains_key("2")); EXPECT_TRUE(c->put("3", "0")); EXPECT_TRUE(c->put("3", "0")); vector keys = {{"3"}, {"4"}}; c->invalidate(keys); EXPECT_FALSE(c->contains_key("3")); EXPECT_FALSE(c->contains_key("3")); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](string const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put("1", string("1"))); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put("2", string("2"))); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](string const& key, SSCCache& c) { loader_called = true; if (key != "99") { EXPECT_TRUE(c.put(key, "99")); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put("3", loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data("4", loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data("99", loader))); // Extra put() overloads c->invalidate(); string vbuf(20, 'v'); EXPECT_TRUE(c->put("1", vbuf.data(), vbuf.size())); val = c->get("1"); EXPECT_EQ(vbuf, *val); c->invalidate(); EXPECT_TRUE(c->put("1", vbuf.data(), vbuf.size(), 'm')); data = c->get_data("1"); EXPECT_EQ(vbuf, data->value); EXPECT_EQ('m', data->metadata); } } // K and M = string TEST(PersistentCache, SDSCache) { unlink_db(test_db); using SDSCache = PersistentCache; { // Constructor and move constructor. auto c = SDSCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); SDSCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = SDSCache::open(test_db); auto c2 = SDSCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = SDSCache::open(test_db); auto val = c->get("1"); EXPECT_FALSE(val); auto data = c->get_data("1"); EXPECT_FALSE(data); auto metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key("1")); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take("42"); EXPECT_FALSE(val); data = c->take_data("42"); EXPECT_FALSE(data); EXPECT_TRUE(c->put("1", 2.0)); val = c->get("1"); EXPECT_EQ(2.0, *val); data = c->get_data("1"); EXPECT_EQ(2.0, data->value); EXPECT_EQ("", data->metadata); metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate("1")); EXPECT_TRUE(c->put("1", 2.0)); data = c->take_data("1"); EXPECT_TRUE(bool(data)); EXPECT_EQ(2.0, data->value); EXPECT_EQ("", data->metadata); EXPECT_TRUE(c->put("2", 3, "4")); data = c->take_data("2"); EXPECT_TRUE(bool(data)); EXPECT_EQ(3, data->value); EXPECT_EQ("4", data->metadata); EXPECT_TRUE(c->put("1", 2, "3")); EXPECT_TRUE(c->put_metadata("1", "3")); data = c->take_data("1"); EXPECT_EQ(2, data->value); EXPECT_EQ("3", data->metadata); val = c->take("42"); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate("42")); EXPECT_FALSE(c->touch("42")); c->invalidate(); c->compact(); EXPECT_TRUE(c->put("1", 0)); EXPECT_TRUE(c->put("2", 0)); c->invalidate({"1", "2"}); EXPECT_FALSE(c->contains_key("1")); EXPECT_FALSE(c->contains_key("2")); EXPECT_TRUE(c->put("3", 0)); EXPECT_TRUE(c->put("3", 0)); vector keys = {{"3"}, {"4"}}; c->invalidate(keys); EXPECT_FALSE(c->contains_key("3")); EXPECT_FALSE(c->contains_key("3")); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](string const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put("1", 1)); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put("2", 2)); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](string const& key, SDSCache& c) { loader_called = true; if (key != "99") { EXPECT_TRUE(c.put(key, 99)); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put("3", loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data("4", loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data("99", loader))); // Extra put() overload c->invalidate(); string mbuf(20, 'm'); EXPECT_TRUE(c->put("1", 2.0, mbuf.data(), mbuf.size())); data = c->get_data("1"); EXPECT_EQ(2.0, data->value); EXPECT_EQ(mbuf, data->metadata); mbuf = string(10, 'x'); EXPECT_TRUE(c->put_metadata("1", mbuf.data(), mbuf.size())); data = c->get_data("1"); EXPECT_EQ(mbuf, data->metadata); } } // V and M = string TEST(PersistentCache, ISSCache) { unlink_db(test_db); using ISSCache = PersistentCache; { // Constructor and move constructor. auto c = ISSCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); ISSCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = ISSCache::open(test_db); auto c2 = ISSCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = ISSCache::open(test_db); auto val = c->get(1); EXPECT_FALSE(val); auto data = c->get_data(1); EXPECT_FALSE(data); auto metadata = c->get_metadata(1); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key(1)); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take(42); EXPECT_FALSE(val); data = c->take_data(42); EXPECT_FALSE(data); EXPECT_TRUE(c->put(1, "2.0")); val = c->get(1); EXPECT_EQ("2.0", *val); data = c->get_data(1); EXPECT_EQ("2.0", data->value); EXPECT_EQ("\0", data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata(1); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate(1)); EXPECT_TRUE(c->put(1, "2.0")); data = c->take_data(1); EXPECT_TRUE(bool(data)); EXPECT_EQ("2.0", data->value); EXPECT_EQ("\0", data->metadata); EXPECT_TRUE(c->put(2, string("3"), "4")); data = c->take_data(2); EXPECT_TRUE(bool(data)); EXPECT_EQ("3", data->value); EXPECT_EQ("4", data->metadata); EXPECT_TRUE(c->put(1, string("2"), "3")); EXPECT_TRUE(c->put_metadata(1, "3")); data = c->take_data(1); EXPECT_EQ("2", data->value); EXPECT_EQ("3", data->metadata); val = c->take(42); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate(42)); EXPECT_FALSE(c->touch(42)); c->invalidate(); c->compact(); EXPECT_TRUE(c->put(1, string("0"))); EXPECT_TRUE(c->put(2, string("0"))); c->invalidate({1, 2}); EXPECT_FALSE(c->contains_key(1)); EXPECT_FALSE(c->contains_key(2)); EXPECT_TRUE(c->put(3, string("0"))); EXPECT_TRUE(c->put(4, string("0"))); vector keys = {3, 4}; c->invalidate(keys); EXPECT_FALSE(c->contains_key(3)); EXPECT_FALSE(c->contains_key(4)); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](int const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put(1, "1")); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put(2, "2")); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](int const& key, ISSCache& c) { loader_called = true; if (key != 99) { EXPECT_TRUE(c.put(key, "99")); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put(3, loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data(4, loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data(99, loader))); // Extra put() overloads c->invalidate(); string vbuf(20, 'v'); string mbuf(20, 'm'); EXPECT_TRUE(c->put(1, vbuf.data(), vbuf.size())); val = c->get(1); EXPECT_EQ(vbuf, *val); c->invalidate(); EXPECT_TRUE(c->put(1, vbuf.data(), vbuf.size(), mbuf.data(), mbuf.size())); data = c->get_data(1); EXPECT_EQ(vbuf, data->value); EXPECT_EQ(mbuf, data->metadata); // Extra put_metadata() overload mbuf = string(10, 'x'); EXPECT_TRUE(c->put_metadata(1, mbuf.data(), mbuf.size())); data = c->get_data(1); EXPECT_EQ(mbuf, data->metadata); } } // K, V and M = string using SSSCache = PersistentCache; TEST(PersistentCache, SSSCache) { unlink_db(test_db); { // Constructor and move constructor. auto c = SSSCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); SSSCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = SSSCache::open(test_db); auto c2 = SSSCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } { auto c = SSSCache::open(test_db); auto val = c->get("1"); EXPECT_FALSE(val); auto data = c->get_data("1"); EXPECT_FALSE(data); auto metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key("1")); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); val = c->take("42"); EXPECT_FALSE(val); data = c->take_data("42"); EXPECT_FALSE(data); EXPECT_TRUE(c->put("1", "2.0")); val = c->get("1"); EXPECT_EQ("2.0", *val); data = c->get_data("1"); EXPECT_EQ("2.0", data->value); EXPECT_EQ("", data->metadata); // decode() generates '\0' for empty metadata metadata = c->get_metadata("1"); EXPECT_FALSE(metadata); // No metadata exists, so false EXPECT_TRUE(c->invalidate("1")); EXPECT_TRUE(c->put("1", "2.0")); data = c->take_data("1"); EXPECT_TRUE(bool(data)); EXPECT_EQ("2.0", data->value); EXPECT_EQ("", data->metadata); EXPECT_TRUE(c->put("2", string("3"), "4")); data = c->take_data("2"); EXPECT_TRUE(bool(data)); EXPECT_EQ("3", data->value); EXPECT_EQ("4", data->metadata); EXPECT_TRUE(c->put("1", string("2"), "3")); EXPECT_TRUE(c->put_metadata("1", "3")); data = c->take_data("1"); EXPECT_EQ("2", data->value); EXPECT_EQ("3", data->metadata); val = c->take("42"); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate("42")); EXPECT_FALSE(c->touch("42")); c->invalidate(); c->compact(); EXPECT_TRUE(c->put("1", "0")); EXPECT_TRUE(c->put("2", "0")); c->invalidate({"1", "2"}); EXPECT_FALSE(c->contains_key("1")); EXPECT_FALSE(c->contains_key("2")); EXPECT_TRUE(c->put("3", "0")); EXPECT_TRUE(c->put("3", "0")); vector keys = {{"3"}, {"4"}}; c->invalidate(keys); EXPECT_FALSE(c->contains_key("3")); EXPECT_FALSE(c->contains_key("3")); c->clear_stats(); c->resize(2048); c->trim_to(0); auto stats = c->stats(); EXPECT_EQ(0, stats.size()); EXPECT_EQ(0, stats.size_in_bytes()); EXPECT_EQ(2048, stats.max_size_in_bytes()); bool handler_called; auto handler = [&](string const&, CacheEvent, PersistentCacheStats const&) { handler_called = true; }; c->set_handler(AllCacheEvents, handler); handler_called = false; EXPECT_TRUE(c->put("1", string("1"))); EXPECT_TRUE(handler_called); c->set_handler(CacheEvent::put, handler); handler_called = false; EXPECT_TRUE(c->put("2", string("2"))); EXPECT_TRUE(handler_called); bool loader_called; auto loader = [&](string const& key, SSSCache& c) { loader_called = true; if (key != "99") { EXPECT_TRUE(c.put(key, "99")); } }; loader_called = false; EXPECT_TRUE(bool(c->get_or_put("3", loader))); loader_called = false; EXPECT_TRUE(bool(c->get_or_put_data("4", loader))); loader_called = false; EXPECT_FALSE(bool(c->get_or_put_data("99", loader))); // Extra put() overloads c->invalidate(); string vbuf(20, 'v'); string mbuf(20, 'm'); EXPECT_TRUE(c->put("1", vbuf.data(), vbuf.size())); val = c->get("1"); EXPECT_EQ(vbuf, *val); c->invalidate(); EXPECT_TRUE(c->put("1", vbuf.data(), vbuf.size(), mbuf.data(), mbuf.size())); data = c->get_data("1"); EXPECT_EQ(vbuf, data->value); EXPECT_EQ(mbuf, data->metadata); // Extra put_metadata() overload mbuf = string(10, 'x'); EXPECT_TRUE(c->put_metadata("1", mbuf.data(), mbuf.size())); data = c->get_data("1"); EXPECT_EQ(mbuf, data->metadata); } } persistent-cache-cpp-1.0.5/tests/core/persistent_string_cache/000077500000000000000000000000001414021052300245315ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/core/persistent_string_cache/CMakeLists.txt000066400000000000000000000010731414021052300272720ustar00rootroot00000000000000add_executable(persistent_string_cache_test persistent_string_cache_test.cpp) target_link_libraries(persistent_string_cache_test ${TESTLIBS}) add_definitions(-DTEST_DIR="${CMAKE_CURRENT_BINARY_DIR}") add_test(persistent_string_cache persistent_string_cache_test) set(TARGETS ${TARGETS} persistent_string_cache_test) add_executable(speed_test speed_test.cpp) target_link_libraries(speed_test ${TESTLIBS}) if (${slowtests}) add_test(speed speed_test) set(TARGETS ${TARGETS} speed_test) endif() set(UNIT_TEST_TARGETS ${UNIT_TEST_TARGETS} ${TARGETS} PARENT_SCOPE) persistent-cache-cpp-1.0.5/tests/core/persistent_string_cache/persistent_string_cache_test.cpp000066400000000000000000000462221414021052300332130ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wctor-dtor-privacy" #include #pragma GCC diagnostic pop #include using namespace std; using namespace core; // Removes the contents of db_dir, but not db_dir itself. void unlink_db(string const& db_dir) { namespace fs = boost::filesystem; try { for (fs::directory_iterator end, it(db_dir); it != end; ++it) { remove_all(it->path()); } } catch (...) { } } string const test_db = TEST_DIR "/db"; TEST(PersistentStringCache, basic) { unlink_db(test_db); { // Constructor and move constructor. auto c = PersistentStringCache::open(test_db, 1024, CacheDiscardPolicy::lru_only); PersistentStringCache c2(move(*c)); EXPECT_EQ(1024, c2.max_size_in_bytes()); } { // Constructor and move assignment. auto c = PersistentStringCache::open(test_db); auto c2 = PersistentStringCache::open(test_db + "2", 2048, CacheDiscardPolicy::lru_ttl); *c = move(*c2); EXPECT_EQ(2048, c->max_size_in_bytes()); } // Tests below are cursory, simply calling each method once. // Note: get_or_put() is tested by PersistentStringCacheImpl_test.cpp. { auto c = PersistentStringCache::open(test_db); Optional val; Optional metadata; Optional data; val = c->get("x"); EXPECT_FALSE(val); data = c->get_data("x"); EXPECT_FALSE(data); metadata = c->get_metadata("x"); EXPECT_FALSE(metadata); EXPECT_FALSE(c->contains_key("x")); EXPECT_EQ(0, c->size()); EXPECT_EQ(0, c->size_in_bytes()); EXPECT_EQ(1024, c->max_size_in_bytes()); EXPECT_GE(c->disk_size_in_bytes(), 0); // For coverage EXPECT_EQ(CacheDiscardPolicy::lru_only, c->discard_policy()); EXPECT_TRUE(c->put("x", "")); EXPECT_TRUE(c->put("x", "x", 1)); EXPECT_TRUE(c->put("x", "y", "")); EXPECT_TRUE(c->put("x", "y", 1, "z", 1)); EXPECT_TRUE(c->put_metadata("x", "z")); EXPECT_TRUE(c->put_metadata("x", "z", 1)); data = c->take_data("x"); EXPECT_TRUE(bool(data)); EXPECT_EQ("y", data->value); EXPECT_EQ("z", data->metadata); PersistentStringCache::Data data2; data2 = *data; // Assignment operator EXPECT_EQ("y", data2.value); EXPECT_EQ("z", data2.metadata); val = c->take("x"); EXPECT_FALSE(val); EXPECT_FALSE(c->invalidate("x")); EXPECT_FALSE(c->touch("x")); c->invalidate(); c->compact(); c->put("x", ""); c->invalidate({"x"}); EXPECT_FALSE(c->contains_key("x")); c->clear_stats(); c->resize(2048); c->trim_to(0); auto handler = [&](string const&, CacheEvent, PersistentCacheStats const&) { }; c->set_handler(AllCacheEvents, handler); c->set_handler(CacheEvent::get, handler); } } TEST(PersistentStringCache, stats) { { PersistentCacheStats s; EXPECT_EQ("", s.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_only, s.policy()); EXPECT_EQ(0, s.size()); EXPECT_EQ(0, s.size_in_bytes()); EXPECT_EQ(0, s.max_size_in_bytes()); EXPECT_EQ(0, s.hits()); EXPECT_EQ(0, s.misses()); EXPECT_EQ(0, s.hits_since_last_miss()); EXPECT_EQ(0, s.misses_since_last_hit()); EXPECT_EQ(0, s.longest_hit_run()); EXPECT_EQ(0, s.longest_miss_run()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); EXPECT_EQ(chrono::system_clock::time_point(), s.most_recent_hit_time()); EXPECT_EQ(chrono::system_clock::time_point(), s.most_recent_miss_time()); EXPECT_EQ(chrono::system_clock::time_point(), s.longest_hit_run_time()); EXPECT_EQ(chrono::system_clock::time_point(), s.longest_miss_run_time()); auto hist = s.histogram(); for (unsigned i = 0; i < hist.size(); ++i) { EXPECT_EQ(0u, hist[i]); // Other bins must still be empty. } } { unlink_db(test_db); auto c = PersistentStringCache::open(test_db, 1024, CacheDiscardPolicy::lru_ttl); // Check that we start out with everything initialized. auto s = c->stats(); EXPECT_EQ(test_db, s.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s.policy()); EXPECT_EQ(0, s.size()); EXPECT_EQ(0, s.size_in_bytes()); EXPECT_EQ(1024, s.max_size_in_bytes()); EXPECT_EQ(0, s.hits()); EXPECT_EQ(0, s.misses()); EXPECT_EQ(0, s.hits_since_last_miss()); EXPECT_EQ(0, s.misses_since_last_hit()); EXPECT_EQ(0, s.longest_hit_run()); EXPECT_EQ(0, s.longest_miss_run()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); EXPECT_EQ(chrono::system_clock::time_point(), s.most_recent_hit_time()); EXPECT_EQ(chrono::system_clock::time_point(), s.most_recent_miss_time()); EXPECT_EQ(chrono::system_clock::time_point(), s.longest_hit_run_time()); EXPECT_EQ(chrono::system_clock::time_point(), s.longest_miss_run_time()); // Incur a miss followed by a hit. auto now = chrono::system_clock::now(); c->put("x", "y"); EXPECT_FALSE(c->get("y")); EXPECT_TRUE(bool(c->get("x"))); s = c->stats(); EXPECT_EQ(test_db, s.cache_path()); // Must not have changed. EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s.policy()); // Must not have changed. EXPECT_EQ(1, s.size()); EXPECT_EQ(2, s.size_in_bytes()); EXPECT_EQ(1024, s.max_size_in_bytes()); // Must not have changed. EXPECT_EQ(1, s.hits()); EXPECT_EQ(1, s.misses()); EXPECT_EQ(1, s.hits_since_last_miss()); EXPECT_EQ(0, s.misses_since_last_hit()); EXPECT_EQ(1, s.longest_hit_run()); EXPECT_EQ(1, s.longest_miss_run()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); EXPECT_NE(chrono::system_clock::time_point(), s.most_recent_hit_time()); EXPECT_GE(s.most_recent_hit_time(), now); EXPECT_NE(chrono::system_clock::time_point(), s.most_recent_miss_time()); EXPECT_GE(s.most_recent_miss_time(), now); EXPECT_NE(chrono::system_clock::time_point(), s.longest_hit_run_time()); EXPECT_GE(s.longest_hit_run_time(), now); EXPECT_NE(chrono::system_clock::time_point(), s.longest_miss_run_time()); EXPECT_GE(s.longest_miss_run_time(), now); auto last_hit_time = s.most_recent_hit_time(); auto last_miss_time = s.most_recent_miss_time(); auto hit_run_time = s.longest_hit_run_time(); auto miss_run_time = s.longest_miss_run_time(); // Two more hits. now = chrono::system_clock::now(); EXPECT_TRUE(bool(c->get("x"))); EXPECT_TRUE(bool(c->get("x"))); s = c->stats(); EXPECT_EQ(3, s.hits()); EXPECT_EQ(1, s.misses()); EXPECT_EQ(3, s.hits_since_last_miss()); EXPECT_EQ(0, s.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(1, s.longest_miss_run()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); EXPECT_LE(last_hit_time, s.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s.most_recent_miss_time()); EXPECT_LE(hit_run_time, s.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s.longest_miss_run_time()); last_hit_time = s.most_recent_hit_time(); last_miss_time = s.most_recent_miss_time(); hit_run_time = s.longest_hit_run_time(); miss_run_time = s.longest_miss_run_time(); // Four more misses now = chrono::system_clock::now(); EXPECT_FALSE(c->get("y")); EXPECT_FALSE(c->get("y")); EXPECT_FALSE(c->get("y")); EXPECT_FALSE(c->get("y")); s = c->stats(); EXPECT_EQ(3, s.hits()); EXPECT_EQ(5, s.misses()); EXPECT_EQ(0, s.hits_since_last_miss()); EXPECT_EQ(4, s.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); EXPECT_EQ(last_hit_time, s.most_recent_hit_time()); EXPECT_LE(last_miss_time, s.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s.longest_hit_run_time()); EXPECT_LE(miss_run_time, s.longest_miss_run_time()); last_hit_time = s.most_recent_hit_time(); last_miss_time = s.most_recent_miss_time(); hit_run_time = s.longest_hit_run_time(); miss_run_time = s.longest_miss_run_time(); // One more hit now = chrono::system_clock::now(); EXPECT_TRUE(bool(c->get("x"))); s = c->stats(); EXPECT_EQ(4, s.hits()); EXPECT_EQ(5, s.misses()); EXPECT_EQ(1, s.hits_since_last_miss()); EXPECT_EQ(0, s.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(0, s.lru_evictions()); EXPECT_LE(last_hit_time, s.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s.most_recent_miss_time()); EXPECT_LE(hit_run_time, s.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s.longest_miss_run_time()); last_hit_time = s.most_recent_hit_time(); last_miss_time = s.most_recent_miss_time(); hit_run_time = s.longest_hit_run_time(); miss_run_time = s.longest_miss_run_time(); { // Copy constructor. auto s2(s); EXPECT_EQ(test_db, s2.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s2.policy()); EXPECT_EQ(1, s2.size()); EXPECT_EQ(2, s2.size_in_bytes()); EXPECT_EQ(1024, s2.max_size_in_bytes()); EXPECT_EQ(4, s2.hits()); EXPECT_EQ(5, s2.misses()); EXPECT_EQ(1, s2.hits_since_last_miss()); EXPECT_EQ(0, s2.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s2.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s2.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s2.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s2.longest_miss_run_time()); } { // Assignment operator. PersistentCacheStats s2; s2 = s; EXPECT_EQ(test_db, s2.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s2.policy()); EXPECT_EQ(1, s2.size()); EXPECT_EQ(2, s2.size_in_bytes()); EXPECT_EQ(1024, s2.max_size_in_bytes()); EXPECT_EQ(4, s2.hits()); EXPECT_EQ(5, s2.misses()); EXPECT_EQ(1, s2.hits_since_last_miss()); EXPECT_EQ(0, s2.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s2.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s2.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s2.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s2.longest_miss_run_time()); } { // Move constructor. PersistentCacheStats s2(move(s)); EXPECT_EQ(test_db, s2.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s2.policy()); EXPECT_EQ(1, s2.size()); EXPECT_EQ(2, s2.size_in_bytes()); EXPECT_EQ(1024, s2.max_size_in_bytes()); EXPECT_EQ(4, s2.hits()); EXPECT_EQ(5, s2.misses()); EXPECT_EQ(1, s2.hits_since_last_miss()); EXPECT_EQ(0, s2.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s2.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s2.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s2.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s2.longest_miss_run_time()); s.cache_path(); // Moved-from instance must be usable. // Move assignment. PersistentCacheStats s3; s3 = move(s2); EXPECT_EQ(test_db, s3.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s3.policy()); EXPECT_EQ(1, s3.size()); EXPECT_EQ(2, s3.size_in_bytes()); EXPECT_EQ(1024, s3.max_size_in_bytes()); EXPECT_EQ(4, s3.hits()); EXPECT_EQ(5, s3.misses()); EXPECT_EQ(1, s3.hits_since_last_miss()); EXPECT_EQ(0, s3.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s3.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s3.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s3.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s3.longest_miss_run_time()); s2.cache_path(); // Moved-from instance must be usable. // Move constructor from internal instance. PersistentCacheStats s4(move(c->stats())); EXPECT_EQ(test_db, s4.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s4.policy()); EXPECT_EQ(1, s4.size()); EXPECT_EQ(2, s4.size_in_bytes()); EXPECT_EQ(1024, s4.max_size_in_bytes()); EXPECT_EQ(4, s4.hits()); EXPECT_EQ(5, s4.misses()); EXPECT_EQ(1, s4.hits_since_last_miss()); EXPECT_EQ(0, s4.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s4.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s4.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s4.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s4.longest_miss_run_time()); s.cache_path(); // Moved-from instance must be usable. } // To get coverage for copying and moving from the internal instance, // we need to use an event handler because the event handler is passed the // shared_ptr to the internal instance, whereas c->stats() returns a copy. auto copy_construct_handler = [&](string const&, CacheEvent, PersistentCacheStats const& s) { PersistentCacheStats s2(s); EXPECT_EQ(test_db, s2.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s2.policy()); EXPECT_EQ(1, s2.size()); EXPECT_EQ(2, s2.size_in_bytes()); EXPECT_EQ(1024, s2.max_size_in_bytes()); EXPECT_EQ(4, s2.hits()); EXPECT_EQ(5, s2.misses()); EXPECT_EQ(1, s2.hits_since_last_miss()); EXPECT_EQ(0, s2.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s2.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s2.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s2.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s2.longest_miss_run_time()); EXPECT_EQ(test_db, s.cache_path()); // Source must remain intact. }; c->set_handler(CacheEvent::touch, copy_construct_handler); c->touch("x"); auto move_assign_handler = [&](string const&, CacheEvent, PersistentCacheStats const& s) { PersistentCacheStats s2; s2 = move(s); EXPECT_EQ(test_db, s2.cache_path()); EXPECT_EQ(CacheDiscardPolicy::lru_ttl, s2.policy()); EXPECT_EQ(1, s2.size()); EXPECT_EQ(2, s2.size_in_bytes()); EXPECT_EQ(1024, s2.max_size_in_bytes()); EXPECT_EQ(4, s2.hits()); EXPECT_EQ(5, s2.misses()); EXPECT_EQ(1, s2.hits_since_last_miss()); EXPECT_EQ(0, s2.misses_since_last_hit()); EXPECT_EQ(3, s.longest_hit_run()); EXPECT_EQ(4, s.longest_miss_run()); EXPECT_EQ(last_hit_time, s2.most_recent_hit_time()); EXPECT_EQ(last_miss_time, s2.most_recent_miss_time()); EXPECT_EQ(hit_run_time, s2.longest_hit_run_time()); EXPECT_EQ(miss_run_time, s2.longest_miss_run_time()); EXPECT_EQ(test_db, s.cache_path()); // Moved-from instance wasn't really moved from. }; c->set_handler(CacheEvent::touch, move_assign_handler); c->touch("x"); // Move construction is impossible because handlers are passed a const ref // to the internal instance. } { unlink_db(test_db); auto c = PersistentStringCache::open(test_db, 1024, CacheDiscardPolicy::lru_ttl); // Check that eviction counts are correct. c->put("1", string(200, 'a')); c->put("2", string(200, 'a')); c->put("3", string(200, 'a')); c->put("4", string(200, 'a')); c->put("5", string(200, 'a')); // Cache almost full now (1005 bytes). Adding a 401-byte record must evict two entries. c->put("6", string(400, 'a')); EXPECT_EQ(1004, c->size_in_bytes()); auto s = c->stats(); EXPECT_EQ(4, s.size()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(2, s.lru_evictions()); // Add two records that expire in 500ms. These must evict two more entries. auto later = chrono::system_clock::now() + chrono::milliseconds(500); c->put("7", string(200, 'a'), later); c->put("8", string(200, 'a'), later); EXPECT_EQ(1004, c->size_in_bytes()); s = c->stats(); EXPECT_EQ(4, s.size()); EXPECT_EQ(0, s.ttl_evictions()); EXPECT_EQ(4, s.lru_evictions()); // Wait until records have expired. while (chrono::system_clock::now() <= later) { this_thread::sleep_for(chrono::milliseconds(5)); } // Add a single record. That must evict both expired entries, even though evicting one would be enough. c->put("9", string(300, 'a')); EXPECT_EQ(903, c->size_in_bytes()); s = c->stats(); EXPECT_EQ(3, s.size()); EXPECT_EQ(2, s.ttl_evictions()); EXPECT_EQ(4, s.lru_evictions()); } } persistent-cache-cpp-1.0.5/tests/core/persistent_string_cache/speed_test.cpp000066400000000000000000000151671414021052300274060ustar00rootroot00000000000000/* * Copyright (C) 2015 Canonical Ltd. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License version 3 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * Authored by: Michi Henning */ #include #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wctor-dtor-privacy" #include #pragma GCC diagnostic pop #include #include #include #include using namespace std; using namespace core; // Removes the contents of db_dir, but not db_dir itself. void unlink_db(string const& db_dir) { namespace fs = boost::filesystem; try { for (fs::directory_iterator end, it(db_dir); it != end; ++it) { remove_all(it->path()); } } catch (...) { } } string const test_db = TEST_DIR "/perf"; char random_char() { static auto seed = random_device()(); static mt19937 engine(seed); static uniform_int_distribution uniform_dist(0, 255); return uniform_dist(engine); } int random_int(int min, int max) { static auto seed = random_device()(); static mt19937 engine(seed); uniform_int_distribution uniform_dist(min, max); return uniform_dist(engine); } int random_size(double mean, double stddev, int64_t min, int64_t max) { static auto seed = random_device()(); static mt19937 engine(seed); normal_distribution normal_dist(mean, stddev); double val = round(normal_dist(engine)); return val < min ? min : (val > max ? max : val); } string const& random_string(int size) { assert(size >= 0); static string s; s.resize(size); for (auto i = 0; i < size; ++i) { s[i] = random_char(); } return s; } // Returns random key padded with zeros to keylen. string make_key(int max_key, int keylen) { static ostringstream s; s << setfill('0') << setw(keylen) << random_int(0, max_key); string key = s.str(); s.clear(); s.str(""); return key; } TEST(PersistentStringCache, basic) { double const kB = 1024.0; double const MB = kB * 1024; // Adjustable parameters int64_t const max_cache_size = 100 * MB; int const value_size = 20 * kB; double const hit_rate = 0.8; int const iterations = 10000; int keylen = 60; double const stddev = value_size / 3.0; auto const cost_of_miss = chrono::microseconds(0); // End adjustable parameters int64_t const num_records = max_cache_size / (keylen + value_size); int const max_key = ((1 - hit_rate) + 1) * num_records - 1; unlink_db(test_db); auto c = PersistentStringCache::open(test_db, max_cache_size, CacheDiscardPolicy::lru_only); cout.setf(ios::fixed, ios::floatfield); cout.precision(3); cout << "Cache size: " << max_cache_size / MB << " MB" << endl; cout << "Records: " << num_records << endl; cout << "Record size: " << value_size / kB << " kB" << endl; cout << "Std. deviation: " << stddev << endl; cout << "Key length: " << keylen << endl; cout << "Hit rate: " << hit_rate << endl; cout << "Cost of miss: " << (cost_of_miss.count() == 0 ? 0.0 : chrono::duration_cast(cost_of_miss).count() / 1000.0) << " ms" << endl; cout << "Iterations: " << iterations << endl; static Optional val; auto start = chrono::system_clock::now(); for (int i = 0; i < num_records; ++i) { static ostringstream s; s << setfill('0') << setw(keylen) << i; string key = s.str(); s.clear(); s.str(""); string const& val = random_string(random_size(value_size, stddev, 0, max_cache_size)); c->put(key, val); } auto now = chrono::system_clock::now(); double secs = chrono::duration_cast(now - start).count() / 1000.0; cout << "Cache full, inserted " << (num_records * value_size) / MB << " MB in " << secs << " seconds (" << c->size_in_bytes() / MB / secs << " MB/sec)" << endl; int64_t bytes_read = 0; int64_t bytes_written = 0; c->clear_stats(); start = chrono::system_clock::now(); for (auto i = 0; i < iterations; ++i) { string key = make_key(max_key, keylen); val = c->get(key); if (!val) { string const& new_val = random_string(random_size(value_size, stddev, 0, max_cache_size)); this_thread::sleep_for(cost_of_miss); c->put(key, new_val); bytes_written += new_val.size(); } else { bytes_read += val->size(); } } now = chrono::system_clock::now(); secs = chrono::duration_cast(now - start).count() / 1000.0; cout << endl; cout << "Performed " << iterations << " lookups with " << hit_rate * 100 << "% hit rate in " << secs << " seconds." << endl; cout << "Read: " << bytes_read / MB << " MB (" << bytes_read / MB / secs << " MB/sec)" << endl; cout << "Wrote: " << bytes_written / MB << " MB (" << bytes_written / MB / secs << " MB/sec)" << endl; auto sum = bytes_read + bytes_written; cout << "Total: " << sum / MB << " MB (" << sum / MB / secs << " MB/sec)" << endl; cout << "Records/sec: " << iterations / secs << endl; auto s = c->stats(); cout << "Hits: " << s.hits() << endl; cout << "Misses: " << s.misses() << endl; cout << "Evictions: " << s.lru_evictions() << endl; cout << "Disk size: " << c->disk_size_in_bytes() / MB << " MB" << endl; cout << endl << "Compacting cache... " << flush; start = chrono::system_clock::now(); c->compact(); now = chrono::system_clock::now(); c.reset(); secs = chrono::duration_cast(now - start).count() / 1000; cout << "done" << endl; cout << "Time: " << secs << " sec" << endl; { auto c = PersistentStringCache::open(test_db); cout << "New size: " << c->disk_size_in_bytes() / MB << " MB" << endl; } unlink_db(test_db); // Reclaim disk space } persistent-cache-cpp-1.0.5/tests/headers/000077500000000000000000000000001414021052300203035ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/headers/CMakeLists.txt000066400000000000000000000025671414021052300230550ustar00rootroot00000000000000# # Test that all header files compile stand-alone and that no public header includes an internal one. # set(root_inc_dir ${CMAKE_SOURCE_DIR}/include) set(subdirs core ) foreach(dir ${subdirs}) string(REPLACE "/" "-" location ${dir}) set(public_inc_dir ${root_inc_dir}/${dir}) set(internal_inc_dir ${public_inc_dir}/internal) if (${slowtests}) # Test that each public header compiles stand-alone. add_test(stand-alone-${location}-headers ${CMAKE_CURRENT_SOURCE_DIR}/compile_headers.py ${public_inc_dir} ${CMAKE_CXX_COMPILER} "${CMAKE_CXX_COMPILER_ARG1} -fPIC -fsyntax-only -I${root_inc_dir} -I${public_inc_dir} ${other_inc_dirs} ${CMAKE_CXX_FLAGS} ${extra_inc_dirs}") # Test that each internal header compiles stand-alone. if (IS_DIRECTORY ${internal_inc_dir}) add_test(stand-alone-${location}-internal-headers ${CMAKE_CURRENT_SOURCE_DIR}/compile_headers.py ${internal_inc_dir} ${CMAKE_CXX_COMPILER} "${CMAKE_CXX_COMPILER_ARG1} -fPIC -fsyntax-only -I${root_inc_dir} -I${internal_inc_dir} ${other_inc_dirs} ${CMAKE_CXX_FLAGS} ${extra_includes}") endif() endif() # Test that no public header includes an internal header add_test(clean-public-${location}-headers ${CMAKE_CURRENT_SOURCE_DIR}/check_public_headers.py ${public_inc_dir}) endforeach() persistent-cache-cpp-1.0.5/tests/headers/check_public_headers.py000077500000000000000000000062221414021052300247700ustar00rootroot00000000000000#! /usr/bin/env python3 # # Copyright (C) 2013 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . # # Authored by: Michi Henning # # # Little helper program to test that public header files don't include internal header files. # # Usage: check_public_headers.py directory # # The directory specifies the location of the header files. All files in that directory ending in .h (but not # in subdirectories) are tested. # import argparse import os import sys import re # # Write the supplied message to stderr, preceded by the program name. # def error(msg): print(os.path.basename(sys.argv[0]) + ": " + msg, file=sys.stderr) # # Write the supplied message to stdout, preceded by the program name. # def message(msg): print(os.path.basename(sys.argv[0]) + ": " + msg) # # For each of the supplied headers, check whether that header includes something in an internal directory. # Return the count of headers that do this. # def test_files(hdr_dir, hdrs): num_errs = 0 for hdr in hdrs: try: hdr_name = os.path.join(hdr_dir, hdr) file = open(hdr_name, 'r', encoding = 'utf=8') except OSError as e: error("cannot open \"" + hdr_name + "\": " + e.strerror) sys.exit(1) include_pat = re.compile(r'#[ \t]*include[ \t]+[<"](.*?)[>"]') lines = file.readlines() line_num = 0 for l in lines: line_num += 1 include_mo = include_pat.match(l) if include_mo: hdr_path = include_mo.group(1) if 'internal/' in hdr_path: num_errs += 1 # Yes, write to stdout because this is expected output message(hdr_name + " includes an internal header at line " + str(line_num) + ": " + hdr_path) return num_errs def run(): # # Parse arguments. # parser = argparse.ArgumentParser(description = 'Test that no public header includes an internal header.') parser.add_argument('dir', nargs = 1, help = 'The directory to look for header files ending in ".h"') args = parser.parse_args() # # Find all the .h files in specified directory and look for #include directives that mention "internal/". # hdr_dir = args.dir[0] try: files = os.listdir(hdr_dir) except OSError as e: error("cannot open \"" + hdr_dir + "\": " + e.strerror) sys.exit(1) hdrs = [hdr for hdr in files if hdr.endswith('.h')] if test_files(hdr_dir, hdrs) != 0: sys.exit(1) # Errors were reported earlier if __name__ == '__main__': run() persistent-cache-cpp-1.0.5/tests/headers/compile_headers.py000077500000000000000000000147471414021052300240200ustar00rootroot00000000000000#! /usr/bin/env python3 # Copyright (C) 2013 Canonical Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Authored by: Michi Henning # # Little helper program to test that header files are stand-alone compilable (and therefore don't depend on # other headers being included first). # # Usage: compile_headers.py directory compiler [compiler_flags] # # The directory specifies the location of the header files. All files in that directory ending in .h (but not # in subdirectories) are tested. # # The compiler argument specifies the compiler to use (such as "gcc"), and the compiler_flags argument (which # must be a single string argument, not a bunch of separate strings) specifies any additional flags, such # as "-I -g". The flags need not include "-c". # # For each header file in the specified directory, the script create a corresponding .cpp that includes the # header file. The .cpp file is created in the current directory (which isn't necessarily the same one as # the directory the header files are in). The script runs the compiler on the generated .cpp file and, if the # compiler returns non-zero exit status, it prints a message (on stdout) reporting the failure. # # The script does not stop if a file fails to compile. If all source files compile successfully, no output (other # than the output from the compiler) is written, and the exit status is zero. If one or more files do not compile, # or there are any other errors, such as not being able to open a file, exit status is non-zero. # # Messages about files that fail to compile are written to stdout. Message about other problems, such as non-existent # files and the like, are written to stderr. # # The compiler's output goes to whatever stream the compiler writes to and is left alone. # import argparse import os import re import shlex import subprocess import sys import tempfile import concurrent.futures, multiprocessing # Additional #defines that should be set before including a header. # This is intended for things such as enabling additional features # that are conditionally compiled in the source. extra_defines =[] # # Write the supplied message to stderr, preceded by the program name. # def error(msg): print(os.path.basename(sys.argv[0]) + ": " + msg, file=sys.stderr) # # Write the supplied message to stdout, preceded by the program name. # def message(msg): print(os.path.basename(sys.argv[0]) + ": " + msg) # # Create a source file in the current directory that includes the specified header, compile it, # and check exit status from the compiler. Throw if the compile command itself fails, # return False if the compile command worked but reported errors, True if the compile succeeded. # def run_compiler(hdr, compiler, copts, verbose, hdr_dir): try: src = tempfile.NamedTemporaryFile(suffix='.cpp', dir='.') # Add any extra defines at the beginning of the temporary file. for flag in extra_defines: src.write(bytes("#define " + flag + "" + "\n", 'UTF-8')) src.write(bytes("#include <" + hdr + ">" + "\n", 'UTF-8')) src.flush() # Need this to make the file visible src_name = os.path.join('.', src.name) if verbose: print(compiler + " -c " + src_name + " " + copts) status = subprocess.call([compiler] + shlex.split(copts) + ["-c", src_name]) if status != 0: message("cannot compile \"" + hdr + "\"") # Yes, write to stdout because this is expected output obj = os.path.splitext(src_name)[0] + ".o" try: os.unlink(obj) except: pass gcov = os.path.splitext(src_name)[0] + ".gcno" try: os.unlink(gcov) except: pass return status == 0 except OSError as e: error(e.strerror) raise # # For each of the supplied headers, create a source file in the current directory that includes the header # and then try to compile the header. Returns normally if all files could be compiled successfully and # throws, otherwise. # def test_files(hdrs, compiler, copts, verbose, hdr_dir): num_errs = 0 executor = concurrent.futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) futures = [executor.submit(run_compiler, h, compiler, copts, verbose, hdr_dir) for h in hdrs] for f in futures: try: if not f.result(): num_errs += 1 except OSError: num_errs += 1 pass # Error reported already if num_errs != 0: msg = str(num_errs) + " file" if num_errs != 1: msg += "s" msg += " failed to compile" message(msg) # Yes, write to stdout because this is expected output sys.exit(1) def run(): # # Parse arguments. # parser = argparse.ArgumentParser(description = 'Test that all headers in the passed directory compile stand-alone.') parser.add_argument('-v', '--verbose', action='store_true', help = 'Trace invocations of the compiler') parser.add_argument('dir', nargs = 1, help = 'The directory to look for header files ending in ".h"') parser.add_argument('compiler', nargs = 1, help = 'The compiler executable, such as "gcc"') parser.add_argument('copts', nargs = '?', default="", help = 'The compiler options (excluding -c), such as "-g -Wall -I." as a single string.') args = parser.parse_args() # # Find all the .h files in specified directory and do the compilation for each one. # hdr_dir = args.dir[0] try: files = os.listdir(hdr_dir) except OSError as e: msg = "cannot open \"" + hdr_dir + "\": " + e.strerror error(msg) sys.exit(1) hdrs = [hdr for hdr in files if hdr.endswith('.h')] try: test_files(hdrs, args.compiler[0], args.copts, args.verbose, hdr_dir) except OSError: sys.exit(1) # Errors were written earlier if __name__ == '__main__': run() persistent-cache-cpp-1.0.5/tests/whitespace/000077500000000000000000000000001414021052300210245ustar00rootroot00000000000000persistent-cache-cpp-1.0.5/tests/whitespace/CMakeLists.txt000066400000000000000000000003141414021052300235620ustar00rootroot00000000000000# # Test that all source files, cmakefiles, etc. do not contain trailing whitespace. # add_test(whitespace ${CMAKE_CURRENT_SOURCE_DIR}/check_whitespace.py ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ) persistent-cache-cpp-1.0.5/tests/whitespace/check_whitespace.py000077500000000000000000000077471414021052300247110ustar00rootroot00000000000000#! /usr/bin/env python3 # Copyright (C) 2013 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # Authored by: Michi Henning # # Little helper program to test that source files do not contain trailing whitespace # or tab indentation. # # Usage: check_whitespace.py directory [ignore_prefix] # # The directory specifies the (recursive) location of the source files. Any # files with a path that starts with ignore_prefix are not checked. This is # useful to exclude files that are generated into the build directory. # # See the file_pat definition below for a list of files that are checked. # import argparse import os import re import sys # Print msg on stderr, preceded by program name and followed by newline def error(msg): print(os.path.basename(sys.argv[0]) + ": " + msg, file=sys.stderr) # Function to raise errors encountered by os.walk def raise_error(e): raise e # Scan lines in file_path for bad whitespace. For each file, # print the line numbers that have whitespace issues whitespace_pat = re.compile(r'.*[ \t]$') tab_indent_pat = re.compile(r'^ *\t') def scan_for_bad_whitespace(file_path): global tab_indent_pat, whitespace_pat errors = [] newlines_at_end = 0 with open(file_path, 'rt', encoding='utf-8') as ifile: for lino, line in enumerate(ifile, start=1): if whitespace_pat.match(line) or tab_indent_pat.match(line): errors.append(lino) if line == "\n" and lino != 1: # Don't complain about empty file with only a single line newlines_at_end += 1 else: newlines_at_end = 0 if 0 < len(errors) <= 10: if len(errors) > 1: plural = 's' else: plural = '' print("%s: bad whitespace in line%s %s" % (file_path, plural, ", ".join((str(i) for i in errors)))) elif errors: print("%s: bad whitespace in multiple lines" % file_path) if newlines_at_end: print("%s: multiple new lines at end of file" % file_path) return bool(errors) or newlines_at_end # Parse args parser = argparse.ArgumentParser(description = 'Test that source files do not contain trailing whitespace.') parser.add_argument('dir', nargs = 1, help = 'The directory to (recursively) search for source files') parser.add_argument('ignore_prefix', nargs = '+', default=None, help = 'Ignore source files with a path that starts with the given prefix.') args = parser.parse_args() # Files we want to check for trailing whitespace. file_pat = r'(.*\.(c|cpp|h|hpp|hh|in|install|js|py|qml|sh)$)|(.*CMakeLists\.txt$)' pat = re.compile(file_pat) # Find all the files with matching file extension in the specified # directory and check them for trailing whitespace. directory = os.path.abspath(args.dir[0]) ignores = args.ignore_prefix and args.ignore_prefix or [] found_whitespace = False try: for root, dirs, files in os.walk(directory, onerror = raise_error): for file in files: path = os.path.join(root, file) ignored = False for ignore in ignores: if ignore and path.startswith(os.path.abspath(ignore)): ignored = True break if not ignored and pat.match(file) and scan_for_bad_whitespace(path): found_whitespace = True except OSError as e: error("cannot create file list for \"" + dir + "\": " + e.strerror) sys.exit(1) if found_whitespace: sys.exit(1) persistent-cache-cpp-1.0.5/valgrind-suppress000066400000000000000000000000001414021052300211270ustar00rootroot00000000000000