pax_global_header 0000666 0000000 0000000 00000000064 13235746211 0014516 g ustar 00root root 0000000 0000000 52 comment=fb792ec353a2fb93f7a61566e1d042fa7811f2a4
.circleci/ 0000775 0000000 0000000 00000000000 13235746211 0012715 5 ustar 00root root 0000000 0000000 .circleci/config.yml 0000664 0000000 0000000 00000022210 13235746211 0014702 0 ustar 00root root 0000000 0000000 version: 2.0
references:
container_config: &container_config
machine: true
cache_init: &cache_init
run:
name: Initialize Cache
command: |
echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}" > /tmp/_build_env_vars
echo Build env vars used for cache keys:
cat /tmp/_build_env_vars
container_setup_pre: &container_setup_pre
restore_cache:
keys:
# Find the most recent cache from any branch
- v4_container_setup_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
container_setup_post: &container_setup_post
save_cache:
# Add _aptcache_contents to cache key so that it is re-uploaded each time the cache changes.
key: v4_container_setup_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ checksum "/tmp/_aptcache_contents" }}
paths:
- /tmp/aptcache
container_setup: &container_setup
run:
name: Setup Environment
command: |
if [ -d "/tmp/aptcache" ]; then
echo Using packages from apt cache
sudo cp -R /tmp/aptcache/* /var/cache/apt/archives/
else
echo No apt cache found
fi
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
sudo touch /etc/apt/sources.list.d/clang.list
sudo chmod o+w /etc/apt/sources.list.d/clang.list
cat > /etc/apt/sources.list.d/clang.list << EOF
deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-3.7 main
deb-src http://apt.llvm.org/trusty/ llvm-toolchain-trusty-3.7 main
deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-3.8 main
deb-src http://apt.llvm.org/trusty/ llvm-toolchain-trusty-3.8 main
deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main
deb-src http://apt.llvm.org/trusty/ llvm-toolchain-trusty-4.0 main
deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main
deb-src http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main
EOF
sudo chmod o-w /etc/apt/sources.list.d/clang.list
DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq
DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git ccache $APT_COMPILER_PACKAGE cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
# Use /dev/urandom when /dev/random is accessed to use less entropy
sudo cp -a /dev/urandom /dev/random
if [ "${BUILD_TOOLSET}" = "clang" ]; then
# They aren't set automatically unfortunately
sudo ln -s /usr/bin/$CC /usr/bin/clang
sudo ln -s /usr/bin/$CXX /usr/bin/clang++
# Need a c++14 compliant STL for clang
sudo apt-get install g++-5
sudo apt-get remove g++-4.8 gcc-4.8
fi
# Setup ccache
sudo ln -s /usr/bin/ccache /usr/local/bin/$CC
sudo ln -s /usr/bin/ccache /usr/local/bin/$CXX
sudo mkdir /ccache_data
sudo chown circleci:circleci /ccache_data
echo 'export CCACHE_COMPILERCHECK=content' >> $BASH_ENV
echo 'export CCACHE_COMPRESS=1' >> $BASH_ENV
echo 'export CCACHE_DIR=/ccache_data' >> $BASH_ENV
echo 'export CCACHE_SLOPPINESS=include_file_mtime' >> $BASH_ENV
sudo mkdir -p /tmp/aptcache
sudo cp -R /var/cache/apt/archives/* /tmp/aptcache/
ls /tmp/aptcache > /tmp/_aptcache_contents
echo
echo System Info:
cat /etc/issue
uname -a
cmake --version
/usr/local/bin/$CC --version
/usr/local/bin/$CXX --version
upgrade_boost_pre: &upgrade_boost_pre
restore_cache:
keys:
# Find the most recent cache from any branch
- v2_upgrade_boost_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
upgrade_boost_post: &upgrade_boost_post
save_cache:
key: v2_upgrade_boost_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
paths:
- /tmp/boost_1_56_0
upgrade_boost: &upgrade_boost
run:
name: Upgrade Boost
command: |
# Detect number of CPU cores
export NUMCORES=`nproc`
echo Using $NUMCORES cores
# Download and prepare boost (only if not already present from cache)
if [ ! -d "/tmp/boost_1_56_0" ]; then
echo "Didn't find boost in cache. Downloading and building."
wget -O /tmp/boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.bz2/download
if [ $(sha512sum /tmp/boost.tar.bz2 | awk '{print $1;}') == "1ce9871c3a2519682538a5f1331938b948123432d99aea0ce026958cbebd25d84019a3a28c452724b9693df98a8d1264bb2d93d2fee508453f8b42836e67481e" ]; then
echo Correct sha512sum
else
echo Wrong sha512sum
sha512sum boost.tar.bz2
exit 1
fi
echo Extracting...
tar -xf /tmp/boost.tar.bz2 -C /tmp
rm -rf boost.tar.bz2
cd /tmp/boost_1_56_0
./bootstrap.sh --with-toolset=${BUILD_TOOLSET} --with-libraries=filesystem,thread,chrono,program_options
cd ..
else
echo Found boost in cache. Use cache and build.
fi
# Compile and install boost (if cached, this should be fast)
cd /tmp/boost_1_56_0
sudo ./b2 toolset=${BUILD_TOOLSET} link=static cxxflags=-fPIC -d0 -j$NUMCORES install
build_pre: &build_pre
restore_cache:
keys:
# Find most recent cache from any revision on the same branch (cache keys are prefix matched)
# CIRCLE_PR_NUMBER is only set if this is a pull request.
- v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ .Branch }}_{{ .Environment.CIRCLE_PR_NUMBER }}
# Fallback to less specific caches if the one above wasn't found
- v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ .Branch }}
- v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
build_post: &build_post
save_cache:
key: v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ .Branch }}_{{ .Environment.CIRCLE_PR_NUMBER }}_{{ .Revision }}
paths:
- /ccache_data
build: &build
run:
name: Build
command: |
export NUMCORES=`nproc` && if [ ! -n "$NUMCORES" ]; then export NUMCORES=`sysctl -n hw.ncpu`; fi
echo Using $NUMCORES cores
# Use ccache
export CXX=/usr/local/bin/$CXX
export CC=/usr/local/bin/$CC
ccache --max-size=512M
ccache --show-stats
# Build
mkdir cmake
cd cmake
cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
make -j$NUMCORES
ccache --show-stats
test: &test
run:
name: Test
command: |
cd cmake
./test/gitversion/gitversion-test
./test/cpp-utils/cpp-utils-test
./test/fspp/fspp-test
./test/parallelaccessstore/parallelaccessstore-test
./test/blockstore/blockstore-test
./test/blobstore/blobstore-test
./test/cryfs/cryfs-test
./test/cryfs-cli/cryfs-cli-test
job_definition: &job_definition
<<: *container_config
steps:
- <<: *cache_init
- <<: *container_setup_pre
- <<: *container_setup
- <<: *container_setup_post
- <<: *upgrade_boost_pre
- <<: *upgrade_boost
- <<: *upgrade_boost_post
- checkout
- <<: *build_pre
- <<: *build
- <<: *build_post
- <<: *test
enable_for_tags: &enable_for_tags
filters:
tags:
only: /.*/
jobs:
gcc_4_8:
<<: *job_definition
environment:
CC: gcc-4.8
CXX: g++-4.8
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-4.8"
gcc_5:
<<: *job_definition
environment:
CC: gcc-5
CXX: g++-5
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-5"
gcc_6:
<<: *job_definition
environment:
CC: gcc-6
CXX: g++-6
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-6"
gcc_7:
<<: *job_definition
environment:
CC: gcc-7
CXX: g++-7
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-7"
clang_3_7:
<<: *job_definition
environment:
CC: clang-3.7
CXX: clang++-3.7
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-3.7
clang_3_8:
<<: *job_definition
environment:
CC: clang-3.8
CXX: clang++-3.8
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-3.8
clang_4_0:
<<: *job_definition
environment:
CC: clang-4.0
CXX: clang++-4.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-4.0
clang_5_0:
<<: *job_definition
environment:
CC: clang-5.0
CXX: clang++-5.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-5.0
workflows:
version: 2
build_and_test:
jobs:
- gcc_4_8:
<<: *enable_for_tags
- gcc_5:
<<: *enable_for_tags
- gcc_6:
<<: *enable_for_tags
- gcc_7:
<<: *enable_for_tags
- clang_3_7:
<<: *enable_for_tags
- clang_3_8:
<<: *enable_for_tags
- clang_4_0:
<<: *enable_for_tags
- clang_5_0:
<<: *enable_for_tags
.gitignore 0000664 0000000 0000000 00000000240 13235746211 0013046 0 ustar 00root root 0000000 0000000 umltest.inner.sh
umltest.status
/build
/cmake
/.idea
*~
src/gitversion/*.pyc
src/gitversion/__pycache__
cmake-build-debug
cmake-build-release
cmake-build-test
.travis.yml 0000664 0000000 0000000 00000003443 13235746211 0013177 0 ustar 00root root 0000000 0000000 language: cpp
sudo: required
dist: trusty
compiler:
- gcc
- clang
os:
- linux
- osx
addons:
apt:
sources:
# Both needed for clang 3.7
- llvm-toolchain-precise-3.7
- ubuntu-toolchain-r-test
packages:
- clang-3.7
- libcrypto++-dev
- libfuse-dev
install:
# Use new clang
- if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
# Detect number of CPU cores
- export NUMCORES=`grep -c ^processor /proc/cpuinfo` && if [ ! -n "$NUMCORES" ]; then export NUMCORES=`sysctl -n hw.ncpu`; fi
- echo Using $NUMCORES cores
# Install dependencies
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./travis.install_boost.sh; fi
- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew cask install osxfuse && brew install cryptopp; fi
# Install run_with_fuse.sh
- mkdir cmake
- cd cmake
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
- chmod +x run_with_fuse.sh
- cmake --version
# Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo cp -a /dev/urandom /dev/random; fi
script:
- cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
- make -j$NUMCORES
- ./test/gitversion/gitversion-test
- ./test/cpp-utils/cpp-utils-test
# TODO Also run on osx once fixed
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./run_with_fuse.sh ./test/fspp/fspp-test || exit 1; fi
- ./test/parallelaccessstore/parallelaccessstore-test
- ./test/blockstore/blockstore-test
- ./test/blobstore/blobstore-test
# TODO Also run on osx once fixed
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./test/cryfs/cryfs-test || exit 1; fi
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./test/cryfs-cli/cryfs-cli-test || exit 1; fi
after_script:
- rm run_with_fuse.sh
CMakeLists.txt 0000664 0000000 0000000 00000001333 13235746211 0013622 0 ustar 00root root 0000000 0000000 cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
project(cryfs)
include(utils.cmake)
require_gcc_version(4.8)
require_clang_version(3.7)
# Default value is not to build test cases
if(NOT BUILD_TESTING)
set(BUILD_TESTING OFF CACHE BOOL "BUILD_TESTING")
endif(NOT BUILD_TESTING)
# Default vaule is to build in release mode
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE INTERNAL "CMAKE_BUILD_TYPE")
endif(NOT CMAKE_BUILD_TYPE)
# Default value is to do update checks
if(NOT CRYFS_UPDATE_CHECKS)
set(CRYFS_UPDATE_CHECKS ON CACHE BOOL "CRYFS_UPDATE_CHECKS")
endif(NOT CRYFS_UPDATE_CHECKS)
add_subdirectory(vendor)
add_subdirectory(src)
add_subdirectory(doc)
add_subdirectory(test)
add_subdirectory(cpack)
ChangeLog.txt 0000664 0000000 0000000 00000021204 13235746211 0013451 0 ustar 00root root 0000000 0000000 Version 0.9.9
--------------
Improvements:
* Add --allow-filesystem-upgrade option which will upgrade old file systems without asking the user. This will be especially helpful for GUI tools.
* Add --version option that shows the CryFS version and exits.
* When CryFS fails to load a file system, the process stops with a helpful error code, which can be used by GUI tools to show detailed messages.
* Only migrate a file system if the underlying storage format changed
Version 0.9.8
--------------
Compatibility:
* Runs on Debian with FreeBSD kernel
* Runs on FreeBSD 11.1
* Works with Crypto++ 6.0
Improvements:
* added a man page
Fixed bugs:
* `du` shows correct file system size
* Updated spdlog dependency to fix build on newer systems
Version 0.9.7
--------------
Compatibility:
* Runs on FreeBSD
* Works with Clang++ 3.8 (Debian experimental or newer Ubuntu systems)
* Works with GCC 7
Version 0.9.6
---------------
Fixed bugs:
* Fix potential deadlock
* Fix potential crash
Improvements:
* Allow building with -DCRYFS_UPDATE_CHECKS=off, which will create an executable with disabled update checks (the alternative to disable them in the environment also still works).
* Automatically disable update checks when running in noninteractive mode.
* More detailed error reporting if key derivation fails
Compatibility:
* Compatible with libcurl version >= 7.50.0, and <= 7.21.6 (tested down to 7.19.0)
* Compatible with Crypto++ 5.6.4
* Compatible with compilers running under hardening-wrapper
Version 0.9.5
---------------
Fixed Bugs:
* Fixed a bug that prevented mounting a file system on Mac OS X.
* File system operations correctly update the timestamps (access time, modification time and status change time).
* Reacts correctly to fsync() and fdatasync() syscalls by flushing the corresponding data to the disk.
Improvements:
* When mounting an old file system, CryFS will ask before migrating it to the newest version.
* Operating system tools like the mount command or /proc/self/mountinfo report correct file system type and also report the base directory.
* Compatibility with GCC 6
Version 0.9.4
---------------
Improvements:
* Ciphertext blocks are split into subdirectories (before, all were on top level) to reduce number of files per directory. Some unix tools don't work well with directories with too many entries.
Fixed Bugs:
* Renaming a file to an existing file (i.e. overwriting an existing file) didn't free the allocated memory for the overwritten file
* Renaming a file to an existing file could hurt an invariant in the directory layout (directory entries have to be sorted) and doing so could cause files to seemingly disappear.
* Fix a potential deadlock in the cache
Compatibility:
* The generated .deb packages work for any Ubuntu/Debian based distribution, but will not install the package source for automatic updates if it's an unsupported operating system.
Version 0.9.3
---------------
New Features:
* The ciphertext block size is configurable. You can use the "--blocksize" command line argument.
If not specified, CryFS will ask you for a block size when creating a file system.
* It's easier for tools and scripts to use CryFS:
If an environment variable CRYFS_FRONTEND=noninteractive is set, we don't ask for options
(but take default values for everything that's not specified on command line).
Furthermore, in noninteractive mode, we won't ask for password confirmation when creating a file system.
The password only has to be sent once to stdin.
* You can disable the automatic update check by setting CRYFS_NO_UPDATE_CHECK=true in your environment.
Fixed Bugs:
* Building CryFS from the GitHub tarball (i.e. when there is no .git directory present) works.
* A bug in the fstat implementation caused problems with some text editors (e.g. nano) falsely thinking a file changed since they opened it.
* When trying to rename a file to an already existing file name, a bug deleted it instead.
* Rename operation allows overwriting existing files, as specified in the rename(2) man page.
Compatibility:
* The generated .deb packages for Debian also work for the Devuan operating system.
Version 0.9.2
---------------
* Experimental support for installing CryFS on Mac OS X using homebrew
(0.9.2 is not released for Linux)
Version 0.9.1
---------------
* Report file system usage statistics to the operating system (e.g. amount of space used). This information can be queried using the 'df' tool on linux. See https://github.com/cryfs/cryfs/commit/68acc27e88ff5209ca55ddb4e91f5a449d77fb54
* Use stronger scrypt parameters when generating the config file key from the user password. This makes it a bit more secure, but also takes a bit longer to load a file system. See https://github.com/cryfs/cryfs/commit/7f1493ab9210319cab008e71d4ee8f4d7d920f39
* Fix a bug where deleting a non-empty directory could leave some blocks over. See https://github.com/cryfs/cryfs/commit/df041ac84511e4560c4f099cd8cc089d08e05737
Version 0.9.0
---------------
(warning) file systems created with earlier CryFS versions are incompatible with this release.
* Fully support file access times
* Fix: Password is read from stdin, not from glibc getpass(). This enables external tools (e.g. GUIs) to pass in the password without problems.
* Remove --extpass parameter, because that encourages tool writers to do bad things like storing a password in a file and using --extpass="cat filename".
The password can now be passed in to stdin without problems, so tools should use that.
* Works with zuluMount GUI, https://mhogomchungu.github.io/zuluCrypt/
* Introduce version flags for file system entities to allow future CryFS versions to be backwards-compatible even if the format changes.
* (for developers) New git repository layout. All subrepositories have been merged to one directory.
* (for developers) Using CMake instead of biicode as build system.
Version 0.8.6
---------------
* Fix a deadlock that was caused when a very high load of parallel resize operations was issued, see https://github.com/cryfs/cryfs/issues/3
* Fix a bug that prevented deleting symlinks, see https://github.com/cryfs/cryfs/issues/2
* Gracefully accept modifications to the file access times instead of failing, although they're not stored yet (they will be stored in 0.9.0). This should fix https://github.com/cryfs/cryfs/issues/4
Version 0.8.5
---------------
* Fix package manager warning when installing the .deb package
* Offer a default configuration when creating new filesystems
* If the given base or mount directory doesn't exist, offer to create them
Version 0.8.4
---------------
* Offering .deb packages for Debian and Ubuntu
* Compatibility with 32bit systems
* Support files larger than 4GB
Version 0.8.3
---------------
* Ask for password confirmation when creating new filesystem
* Check for new CryFS versions and ask the user to update if a new version is available
* Implemented a mechanism that can show warnings about security bugs to users of a certain CryFS version. Let's hope this won't be necessary ;)
* Compatibility with GCC 4.8 (that allows compiling on Ubuntu 14.04 for example)
Version 0.8.2
---------------
* Mount directory, base directory, logfile and config file can be specified as relative paths
* Improved error messages
Version 0.8.1
---------------
* Config File Encryption: Configuration files are encrypted with two ciphers. The user specifies a password, which is then used with the scrypt KDF to generate the two encryption keys.
- Inner level: Encrypts the config data using the user specified cipher.
- Outer level: Encrypts the name of the inner cipher and the inner level ciphertext using aes-256-gcm.
The config file is padded to hide the size of the configuration data (including the name of the cipher used).
* No external config file needed: If the configuration file is not specified as command line parameter, it will be put into the base directory. This way, the filesystem can be mounted with the password only, without specifying a config file on command line.
* Logfiles: Added a --logfile option to specify where logs should be written to. If the option is not specified, CryFs logs to syslog.
* Running in Background: Fixed daemonization. When CryFs is run without "-f" flag, it will run in background.
* Better error messages when base directory is not existing, not readable or not writeable.
* Allow --cipher=xxx to specify cipher on command line. If cryfs is creating a new filesystem, it will use this cipher. If it is opening an existing filesystem, it will check whether this is the cipher used by it.
* --show-ciphers shows a list of all supported ciphers
* --extpass allows using an external program for password input
* --unmount-idle x automatically unmounts the filesystem after x minutes without a filesystem operation.
LICENSE 0000664 0000000 0000000 00000016743 13235746211 0012102 0 ustar 00root root 0000000 0000000 GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc.
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
README.md 0000664 0000000 0000000 00000010716 13235746211 0012346 0 ustar 00root root 0000000 0000000 # CryFS [](https://travis-ci.org/cryfs/cryfs)
CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.
See [https://www.cryfs.org](https://www.cryfs.org).
Install latest release
======================
This only works for Ubuntu 17.04 and later, and Debian Stretch and later.
You can also use CryFS on older versions of these distributions by following the **Building from source** instructions below.
sudo apt install cryfs
GUI
===
Theres some GUI applications with CryFS support. You usually have to install the GUI **and** also CryFS itself for it to work.
- [SiriKali](https://mhogomchungu.github.io/sirikali/)
- [Plasma Vault](https://www.kde.org/announcements/plasma-5.11.0.php) in KDE Plasma >= 5.11
Building from source
====================
Requirements
------------
- Git (for getting the source code)
- GCC version >= 4.8 or Clang >= 3.7
- CMake version >= 2.8
- libcurl4 (including development headers)
- Boost libraries version >= 1.56 (including development headers)
- filesystem
- system
- chrono
- program_options
- thread
- Crypto++ version >= 5.6.3 (including development headers)
- SSL development libraries (including development headers, e.g. libssl-dev)
- libFUSE version >= 2.8.6 (including development headers), on Mac OS X instead install osxfuse from https://osxfuse.github.io/
- Python >= 2.7
You can use the following commands to install these requirements
# Ubuntu
$ sudo apt-get install git g++ cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
# Fedora
sudo dnf install git gcc-c++ cmake make libcurl-devel boost-devel boost-static cryptopp-devel openssl-devel fuse-devel python
# Macintosh
brew install cmake boost cryptopp openssl
Build & Install
---------------
1. Clone repository
$ git clone https://github.com/cryfs/cryfs.git cryfs
$ cd cryfs
2. Build
$ mkdir cmake && cd cmake
$ cmake ..
$ make
3. Install
$ sudo make install
You can pass the following variables to the *cmake* command (using *-Dvariablename=value*):
- **-DCMAKE_BUILD_TYPE**=[Release|Debug]: Whether to run code optimization or add debug symbols. Default: Release
- **-DBUILD_TESTING**=[on|off]: Whether to build the test cases (can take a long time). Default: off
- **-DCRYFS_UPDATE_CHECKS**=off: Build a CryFS that doesn't check online for updates and security vulnerabilities.
Troubleshooting
---------------
On most systems, CMake should find the libraries automatically. However, that doesn't always work.
1. **Boost headers not found**
Pass in the boost include path with
cmake .. -DBoost_INCLUDE_DIRS=/path/to/boost/headers
If you want to link boost dynamically (e.g. you don't have the static libraries), use the following:
cmake .. -DBoost_USE_STATIC_LIBS=off
2. **Fuse/Osxfuse library not found**
Pass in the library path with
cmake .. -DFUSE_LIB_PATH=/path/to/fuse/or/osxfuse
3. **Fuse/Osxfuse headers not found**
Pass in the include path with
cmake .. -DCMAKE_CXX_FLAGS="-I/path/to/fuse/or/osxfuse/headers"
4. **CryptoPP library not found**
Pass in the library path with
cmake .. -DCRYPTOPP_LIB_PATH=/path/to/cryptopp
5. **Openssl headers not found**
Pass in the include path with
cmake .. -DCMAKE_C_FLAGS="-I/path/to/openssl/include"
Creating .deb and .rpm packages
-------------------------------
There are additional requirements if you want to create packages. They are:
- CMake version >= 3.3
- rpmbuild for creating .rpm package
1. Clone repository
$ git clone https://github.com/cryfs/cryfs.git cryfs
$ cd cryfs
2. Build
$ mkdir cmake && cd cmake
$ cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=off
$ make package
Disclaimer
----------------------
On the event of a password leak, you are strongly advised to create a new filesystem and copy all the data over from the previous one. Done this, all copies of the compromised filesystem and config file must be removed (e.g, from the "previous versions" feature of your cloud system) to prevent access to the key (and, as a result, your data) using the leaked password.
archive.sh 0000775 0000000 0000000 00000000420 13235746211 0013036 0 ustar 00root root 0000000 0000000 #!/bin/bash
TAG=$1
GPGHOMEDIR=$2
git archive --format=tgz "$1" > cryfs-$1.tar.gz
gpg --homedir "$GPGHOMEDIR" --armor --detach-sign cryfs-$1.tar.gz
git archive --format=tar "$1" | xz -9 > cryfs-$1.tar.xz
gpg --homedir "$GPGHOMEDIR" --armor --detach-sign cryfs-$1.tar.xz
cpack/ 0000775 0000000 0000000 00000000000 13235746211 0012143 5 ustar 00root root 0000000 0000000 cpack/CMakeLists.txt 0000664 0000000 0000000 00000007103 13235746211 0014704 0 ustar 00root root 0000000 0000000 # We only allow generating distribution packages if:
# - it is a release build (to ensure all generated debian packages are fine to be distributed)
# - tests are not built (because otherwise CPack would start installing googletest with the package as well)
string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILDTYPE)
if(BUILDTYPE MATCHES RELEASE AND NOT BUILD_TESTING)
if("${CMAKE_VERSION}" VERSION_LESS "3.3")
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
message(WARNING "Distribution package generation is only supported for CMake version >= 3.3. You're using ${CMAKE_VERSION}. You will be able to build and install CryFS, but you won't be able to generate .deb packages.")
else("${CMAKE_VERSION}" VERSION_LESS "3.3")
# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
# When installing the .deb package and these files have the wrong permissions, the package manager complains.
execute_process(COMMAND /bin/bash -c "chmod 0755 ${CMAKE_CURRENT_SOURCE_DIR}/debfiles/*")
set(CPACK_GENERATOR TGZ DEB RPM)
set(CPACK_PACKAGE_NAME "cryfs")
get_git_version(GITVERSION_VERSION_STRING)
set(CPACK_PACKAGE_VERSION "${GITVERSION_VERSION_STRING}")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Encrypt your files and store them in the cloud.")
set(CPACK_PACKAGE_DESCRIPTION "CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.")
set(CPACK_PACKAGE_CONTACT "Sebastian Messmer ")
set(CPACK_PACKAGE_VENDOR "Sebastian Messmer")
set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE")
set(CPACK_PACKAGE_INSTALL_DIRECTORY "CMake ${CMake_VERSION_MAJOR}.${CMake_VERSION_MINOR}")
if(WIN32 AND NOT UNIX)
# There is a bug in NSI that does not handle full unix paths properly. Make
# sure there is at least one set of four (4) backlasshes.
#set(CPACK_PACKAGE_ICON "${CMake_SOURCE_DIR}/Utilities/Release\\\\InstallIcon.bmp")
#set(CPACK_NSIS_INSTALLED_ICON_NAME "bin\\\\cryfs.exe")
#set(CPACK_NSIS_DISPLAY_NAME "${CPACK_PACKAGE_INSTALL_DIRECTORY} CryFS")
#set(CPACK_NSIS_HELP_LINK "http:\\\\\\\\www.cryfs.org")
#set(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\www.cryfs.org")
#set(CPACK_NSIS_CONTACT "messmer@cryfs.org")
#set(CPACK_NSIS_MODIFY_PATH ON)
else(WIN32 AND NOT UNIX)
set(CPACK_STRIP_FILES ON)
set(CPACK_SOURCE_STRIP_FILES ON)
endif(WIN32 AND NOT UNIX)
set(CPACK_PACKAGE_EXECUTABLES "cryfs" "CryFS")
set(CPACK_DEBIAN_PACKAGE_SECTION "utils")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
# Needs gnupg2, lsb-release for postinst script
set(CPACK_DEBIAN_PACKAGE_DEPENDS "fuse, gnupg2, lsb-release")
set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://www.cryfs.org")
set(CPACK_RPM_PACKAGE_LICENSE "LGPLv3")
set(CPACK_RPM_PACKAGE_DESCRIPTION ${CPACK_PACKAGE_DESCRIPTION})
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/bin;/usr/share/man;/usr/share/man/man1")
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postinst;${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postrm")
include(CPack)
endif("${CMAKE_VERSION}" VERSION_LESS "3.3")
endif(BUILDTYPE MATCHES RELEASE AND NOT BUILD_TESTING)
cpack/debfiles/ 0000775 0000000 0000000 00000000000 13235746211 0013720 5 ustar 00root root 0000000 0000000 cpack/debfiles/postinst 0000775 0000000 0000000 00000011316 13235746211 0015533 0 ustar 00root root 0000000 0000000 #!/bin/bash
# This script is called after the cryfs .deb package is installed.
# It sets up the package source so the user gets automatic updates for cryfs.
# DEVELOPER WARNING: There is a lot of redundancy between this file and the install.sh script in the cryfs-web repository. Please port modifications to there!
set -e
DEBIAN_REPO_URL="http://apt.cryfs.org/debian"
UBUNTU_REPO_URL="http://apt.cryfs.org/ubuntu"
DISTRIBUTION=`lsb_release -s -i`
DISTRIBUTION_VERSION=`lsb_release -s -c`
containsElement () {
local e
for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
return 1
}
get_repo_url () {
if [[ "$DISTRIBUTION" == "Debian" ]] || [[ "$DISTRIBUTION" == "Devuan" ]]; then
echo $DEBIAN_REPO_URL
elif [[ "$DISTRIBUTION" == "Ubuntu" ]]; then
echo $UBUNTU_REPO_URL
else
echo Not adding package source because $DISTRIBUTION is not supported. Please keep CryFS manually up to date. 1>&2
exit 0
fi
}
get_apt_config () {
apt-config dump|grep "$1 "|sed -e "s/^$1\ \"\([^\"]*\)\"\;/\1/g"
}
sources_list_dir () {
root=$(get_apt_config "Dir")
etc=$(get_apt_config "Dir::Etc")
sourceparts=$(get_apt_config "Dir::Etc::sourceparts")
echo "$root/$etc/$sourceparts"
}
add_repository () {
dir=$(sources_list_dir)
repo_url=$(get_repo_url)
echo "deb $repo_url $DISTRIBUTION_VERSION main" > $dir/cryfs.list
}
install_key () {
# Key from http://www.cryfs.org/apt.key
apt-key add - > /dev/null <&2
exit 1
;;
esac
set +e
exit 0
cpack/debfiles/postrm 0000775 0000000 0000000 00000001546 13235746211 0015200 0 ustar 00root root 0000000 0000000 #!/bin/bash
# This script is called after the cryfs .deb package is uninstalled.
# It removes the package source that was used to get automatic updates.
set -e
get_apt_config () {
apt-config dump|grep "$1 "|sed -e "s/^$1\ \"\([^\"]*\)\"\;/\1/g"
}
sources_list_dir () {
root=$(get_apt_config "Dir")
etc=$(get_apt_config "Dir::Etc")
sourceparts=$(get_apt_config "Dir::Etc::sourceparts")
echo $root$etc$sourceparts
}
remove_repository () {
dir=$(sources_list_dir)
rm -f $dir/cryfs.list
}
remove_key () {
# Don't fail if key was already removed
apt-key rm 549E65B2 2>&1 > /dev/null || true
}
case "$1" in
purge)
remove_repository
remove_key
;;
abort-install|abort-upgrade|remove|upgrade|failed-upgrade)
;;
*)
echo "postrm called with unknown argument '$1'" >&2
exit 1
;;
esac
set +e
exit 0
doc/ 0000775 0000000 0000000 00000000000 13235746211 0011627 5 ustar 00root root 0000000 0000000 doc/CMakeLists.txt 0000664 0000000 0000000 00000000676 13235746211 0014400 0 ustar 00root root 0000000 0000000 project (doc)
INCLUDE(GNUInstallDirs)
find_program(GZIP gzip)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
COMMAND ${GZIP} -c ${CMAKE_CURRENT_SOURCE_DIR}/man/cryfs.1 > ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
)
add_custom_target(man ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
DESTINATION ${CMAKE_INSTALL_MANDIR}/man1
CONFIGURATIONS Release
)
doc/man/ 0000775 0000000 0000000 00000000000 13235746211 0012402 5 ustar 00root root 0000000 0000000 doc/man/cryfs.1 0000664 0000000 0000000 00000012533 13235746211 0013616 0 ustar 00root root 0000000 0000000 .\" cryfs(1) man page
.
.TH cryfs 1
.
.
.
.SH NAME
cryfs \- cryptographic filesystem for the cloud
.
.
.
.SH SYNOPSIS
.\" mount/create syntax
.B cryfs
[\fB\-c\fR \fIfile\fR]
[\fB\-f\fR]
[\fIoptions\fR]
.I basedir mountpoint
[\fB\-\-\fR \fIfuse-options\fR]
.br
.\" show-ciphers syntax
.B cryfs \-\-help\fR|\fB\-\-version\fR|\fB\-\-show-ciphers
.
.
.
.SH DESCRIPTION
.
.B CryFS
encrypts your files, so you can safely store them anywhere.
.PP
.
The goal of CryFS is not only to keep file contents, but also
file sizes, metadata and directory structure confidential.
CryFS uses
.B encrypted same-size blocks
to store both the files themselves and the block's relations to another.
These blocks are stored as individual files in the base directory,
which can then be synchronized with cloud services such as Dropbox.
.PP
.
The blocks are encrypted using a random key, which is stored in a
.B configuration file
encrypted by the user's passphrase.
By default, it will be stored together with the data in the base directory,
but you can choose a different location if you do not want it in your cloud
or when using a weak passphrase.
.
.
.
.SH USING CRYFS
.
.SS Selecting base and mount directories
.
While you can access your files through your
.B mount directory,
CryFS actually places them in your
.B base directory
after encrypting.
CryFS will encrypt and decrypt your files 'on the fly' as they are accessed,
so files will never be stored on the disk in unencrypted form.
.PP
.
You can choose any empty directory as your base, but your mount directory
should be outside of any cloud storage, as your cloud may try to sync your
(temporarily mounted) unencrypted files as well.
.
.SS Setup and usage of your encrypted directory
.
.TP
Creating and mounting your encrypted storage use the same command-line syntax:
.B cryfs
.I basedir mountpoint
.PP
.
If CryFS detects an encrypted storage in the given base directory, you will
be asked for the passphrase to unlock and mount it. Otherwise, CryFS will
help you with creating one, just follow the on-screen instructions.
.PP
.
.TP
After you are done working with your encrypted files, unmount your storage \
with the command
.B fusermount -u
.I mountpoint
.
.
.SS Changing your passphrase
.
As the encryption key to your CryFS storage is stored in your configuration
file, it would be possible to re-encrypt it using a different passphrase
(although this feature has not been implemented yet).
.PP
.
However, this does not change the actual encryption key of your storage, so
someone with access to the old passphrase and configuration file (for example
through the file history of your cloud or your file system) could still access
your files, even those created after the password change.
.PP
.
For this reason, the recommended way to change your passphrase is to create a
new CryFS storage with the new passphrase and move your files from the old to
the new one.
.
.
.
.SH OPTIONS
.
.SS Getting help
.
.TP
\fB\-h\fR, \fB\-\-help\fR
.
Show a help message containing short descriptions for all options.
.
.
.TP
\fB\-\-show\-ciphers\fR
.
Show a list of all supported encryption ciphers.
.
.
.TP
\fB\-\-version\fR
.
Show the CryFS version number.
.
.
.SS Encryption parameters
.
.TP
\fB\-\-blocksize\fR \fIarg\fR
.
Set the block size to \fIarg\fR bytes. Defaults to
.BR 32768 .
.br
\" Intentional space
.br
A higher block size may help reducing the file count in your base directory
(especially when storing large files), but will also waste more space when
storing smaller files.
.
.
.TP
\fB\-\-cipher\fR \fIarg\fR
.
Use \fIarg\fR as the cipher for the encryption. Defaults to
.BR aes-256-gcm .
.
.
.TP
\fB\-c\fR \fIfile\fR, \fB\-\-config\fR \fIfile\fR
.
Use \fIfile\fR as configuration file for this CryFS storage instead of
\fIbasedir\fR/cryfs.config
.
.
.SS General options
.
.TP
\fB\-f\fR, \fB\-\-foreground\fI
.
Run CryFS in the foreground. Stop using CTRL-C.
.
.
.TP
\fB\-\-allow-filesystem-upgrade\fI
.
Allow upgrading the file system if it was created with an old CryFS version. After the upgrade, older CryFS versions might not be able to use the file system anymore.
.
.
.TP
\fB\-\-logfile\fR \fIfile\fR
.
Write status information to \fIfile\fR. If no logfile is given, CryFS will
write them to syslog in background mode, or to stdout in foreground mode.
.
.
.TP
\fB\-\-unmount\-idle\fR \fIarg\fR
.
Unmount automatically after \fIarg\fR minutes of inactivity.
.
.
.
.SH ENVIRONMENT
.
.TP
\fBCRYFS_FRONTEND\fR=noninteractive
.
With this option set, CryFS will only ask for the encryption passphrase once.
Instead of asking the user for parameters not specified on the command line,
it will just use the default values. CryFS will also not ask you to confirm
your passphrase when creating a new CryFS storage.
.br
\" Intentional space
.br
Set this environment variable when automating CryFS using external tools or
shell scripts.
.
.
.TP
\fBCRYFS_NO_UPDATE_CHECK\fR=true
.
By default, CryFS connects to the internet to check for known security
vulnerabilities and new versions. This option disables this.
.
.
.
.SH SEE ALSO
.
.BR mount.fuse (1),
.BR fusermount (1)
.PP
.
For more information about the design of CryFS, visit
.B https://www.cryfs.org
.PP
.
Visit the development repository at
.B https://github.com/cryfs/cryfs
for the source code and the full list of contributors to CryFS.
.
.
.
.SH AUTHORS
.
CryFS was created by Sebastian Messmer and contributors.
This man page was written by Maximilian Wende.
src/ 0000775 0000000 0000000 00000000000 13235746211 0011651 5 ustar 00root root 0000000 0000000 src/CMakeLists.txt 0000664 0000000 0000000 00000000425 13235746211 0014412 0 ustar 00root root 0000000 0000000 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(gitversion)
add_subdirectory(cpp-utils)
add_subdirectory(fspp)
add_subdirectory(parallelaccessstore)
add_subdirectory(blockstore)
add_subdirectory(blobstore)
add_subdirectory(cryfs)
add_subdirectory(cryfs-cli)
src/blobstore/ 0000775 0000000 0000000 00000000000 13235746211 0013644 5 ustar 00root root 0000000 0000000 src/blobstore/CMakeLists.txt 0000664 0000000 0000000 00000002131 13235746211 0016401 0 ustar 00root root 0000000 0000000 project (blobstore)
set(SOURCES
implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp
implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp
implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
implementations/onblocks/utils/Math.cpp
implementations/onblocks/BlobStoreOnBlocks.cpp
implementations/onblocks/datanodestore/DataNode.cpp
implementations/onblocks/datanodestore/DataLeafNode.cpp
implementations/onblocks/datanodestore/DataInnerNode.cpp
implementations/onblocks/datanodestore/DataNodeStore.cpp
implementations/onblocks/datatreestore/impl/algorithms.cpp
implementations/onblocks/datatreestore/DataTree.cpp
implementations/onblocks/datatreestore/DataTreeStore.cpp
implementations/onblocks/BlobOnBlocks.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils blockstore)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})
src/blobstore/implementations/ 0000775 0000000 0000000 00000000000 13235746211 0017054 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/ 0000775 0000000 0000000 00000000000 13235746211 0020666 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/BlobOnBlocks.cpp 0000664 0000000 0000000 00000010214 13235746211 0023701 0 ustar 00root root 0000000 0000000 #include "parallelaccessdatatreestore/DataTreeRef.h"
#include "BlobOnBlocks.h"
#include "datanodestore/DataLeafNode.h"
#include "utils/Math.h"
#include
#include
using std::function;
using cpputils::unique_ref;
using cpputils::Data;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
using parallelaccessdatatreestore::DataTreeRef;
BlobOnBlocks::BlobOnBlocks(unique_ref datatree)
: _datatree(std::move(datatree)), _sizeCache(boost::none) {
}
BlobOnBlocks::~BlobOnBlocks() {
}
uint64_t BlobOnBlocks::size() const {
if (_sizeCache == boost::none) {
_sizeCache = _datatree->numStoredBytes();
}
return *_sizeCache;
}
void BlobOnBlocks::resize(uint64_t numBytes) {
_datatree->resizeNumBytes(numBytes);
_sizeCache = numBytes;
}
void BlobOnBlocks::traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function func) const {
uint64_t endByte = beginByte + sizeBytes;
uint32_t firstLeaf = beginByte / _datatree->maxBytesPerLeaf();
uint32_t endLeaf = utils::ceilDivision(endByte, _datatree->maxBytesPerLeaf());
bool writingOutside = size() < endByte; // TODO Calling size() is slow because it has to traverse the tree
_datatree->traverseLeaves(firstLeaf, endLeaf, [&func, beginByte, endByte, endLeaf, writingOutside](DataLeafNode *leaf, uint32_t leafIndex) {
uint64_t indexOfFirstLeafByte = leafIndex * leaf->maxStoreableBytes();
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(leaf->maxStoreableBytes(), endByte - indexOfFirstLeafByte);
if (leafIndex == endLeaf-1 && writingOutside) {
// If we are traversing an area that didn't exist before, then the last leaf was just created with a wrong size. We have to fix it.
leaf->resize(dataEnd);
}
func(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
});
if (writingOutside) {
ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
_sizeCache = endByte;
}
}
Data BlobOnBlocks::readAll() const {
//TODO Querying size is inefficient. Is this possible without a call to size()?
uint64_t count = size();
Data result(count);
_read(result.data(), 0, count);
return result;
}
void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
ASSERT(offset <= size() && offset + count <= size(), "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
uint64_t read = tryRead(target, offset, count);
ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
}
uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
//TODO Quite inefficient to call size() here, because that has to traverse the tree
uint64_t realCount = std::max(UINT64_C(0), std::min(count, size()-offset));
_read(target, offset, realCount);
return realCount;
}
void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
traverseLeaves(offset, count, [target, offset] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
}
void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
traverseLeaves(offset, count, [source, offset] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
}
void BlobOnBlocks::flush() {
_datatree->flush();
}
const Key &BlobOnBlocks::key() const {
return _datatree->key();
}
unique_ref BlobOnBlocks::releaseTree() {
return std::move(_datatree);
}
}
}
src/blobstore/implementations/onblocks/BlobOnBlocks.h 0000664 0000000 0000000 00000002670 13235746211 0023355 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#include "../../interface/Blob.h"
#include
#include
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataLeafNode;
}
namespace parallelaccessdatatreestore {
class DataTreeRef;
}
class BlobOnBlocks final: public Blob {
public:
BlobOnBlocks(cpputils::unique_ref datatree);
~BlobOnBlocks();
const blockstore::Key &key() const override;
uint64_t size() const override;
void resize(uint64_t numBytes) override;
cpputils::Data readAll() const override;
void read(void *target, uint64_t offset, uint64_t size) const override;
uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
cpputils::unique_ref releaseTree();
private:
void _read(void *target, uint64_t offset, uint64_t count) const;
void traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function) const;
cpputils::unique_ref _datatree;
mutable boost::optional _sizeCache;
DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
};
}
}
#endif
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp 0000664 0000000 0000000 00000004366 13235746211 0024731 0 ustar 00root root 0000000 0000000 #include "parallelaccessdatatreestore/DataTreeRef.h"
#include "parallelaccessdatatreestore/ParallelAccessDataTreeStore.h"
#include
#include "datanodestore/DataLeafNode.h"
#include "datanodestore/DataNodeStore.h"
#include "datatreestore/DataTreeStore.h"
#include "datatreestore/DataTree.h"
#include "BlobStoreOnBlocks.h"
#include "BlobOnBlocks.h"
#include
#include
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::BlockStore;
using blockstore::parallelaccess::ParallelAccessBlockStore;
using blockstore::Key;
using cpputils::dynamic_pointer_move;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
using datanodestore::DataNodeStore;
using datatreestore::DataTreeStore;
using parallelaccessdatatreestore::ParallelAccessDataTreeStore;
BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref blockStore, uint64_t physicalBlocksizeBytes)
: _dataTreeStore(make_unique_ref(make_unique_ref(make_unique_ref(make_unique_ref(std::move(blockStore)), physicalBlocksizeBytes)))) {
}
BlobStoreOnBlocks::~BlobStoreOnBlocks() {
}
unique_ref BlobStoreOnBlocks::create() {
return make_unique_ref(_dataTreeStore->createNewTree());
}
optional> BlobStoreOnBlocks::load(const Key &key) {
auto tree = _dataTreeStore->load(key);
if (tree == none) {
return none;
}
return optional>(make_unique_ref(std::move(*tree)));
}
void BlobStoreOnBlocks::remove(unique_ref blob) {
auto _blob = dynamic_pointer_move(blob);
ASSERT(_blob != none, "Passed Blob in BlobStoreOnBlocks::remove() is not a BlobOnBlocks.");
_dataTreeStore->remove((*_blob)->releaseTree());
}
uint64_t BlobStoreOnBlocks::virtualBlocksizeBytes() const {
return _dataTreeStore->virtualBlocksizeBytes();
}
uint64_t BlobStoreOnBlocks::numBlocks() const {
return _dataTreeStore->numNodes();
}
uint64_t BlobStoreOnBlocks::estimateSpaceForNumBlocksLeft() const {
return _dataTreeStore->estimateSpaceForNumNodesLeft();
}
}
}
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h 0000664 0000000 0000000 00000002644 13235746211 0024373 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
#include "../../interface/BlobStore.h"
#include "BlobOnBlocks.h"
#include
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStore;
}
//TODO Make blobstore able to cope with incomplete data (some blocks missing, because they're not synchronized yet) and write test cases for that
class BlobStoreOnBlocks final: public BlobStore {
public:
BlobStoreOnBlocks(cpputils::unique_ref blockStore, uint64_t physicalBlocksizeBytes);
~BlobStoreOnBlocks();
cpputils::unique_ref create() override;
boost::optional> load(const blockstore::Key &key) override;
void remove(cpputils::unique_ref blob) override;
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
//virtual means "space we can use" as opposed to "space it takes on the disk" (i.e. virtual is without headers, checksums, ...)
uint64_t virtualBlocksizeBytes() const override;
uint64_t numBlocks() const override;
uint64_t estimateSpaceForNumBlocksLeft() const override;
private:
cpputils::unique_ref _dataTreeStore;
DISALLOW_COPY_AND_ASSIGN(BlobStoreOnBlocks);
};
}
}
#endif
src/blobstore/implementations/onblocks/datanodestore/ 0000775 0000000 0000000 00000000000 13235746211 0023522 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp 0000664 0000000 0000000 00000005445 13235746211 0026711 0 ustar 00root root 0000000 0000000 #include "DataInnerNode.h"
#include "DataNodeStore.h"
#include
using blockstore::Block;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataInnerNode::DataInnerNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(depth() > 0, "Inner node can't have depth 0. Is this a leaf maybe?");
if (node().FormatVersion() != FORMAT_VERSION_HEADER) {
throw std::runtime_error("This node format is not supported. Was it created with a newer version of CryFS?");
}
}
DataInnerNode::~DataInnerNode() {
}
unique_ref DataInnerNode::InitializeNewNode(unique_ref block, const DataNode &first_child) {
DataNodeView node(std::move(block));
node.setFormatVersion(DataNode::FORMAT_VERSION_HEADER);
node.setDepth(first_child.depth() + 1);
node.setSize(1);
auto result = make_unique_ref(std::move(node));
result->ChildrenBegin()->setKey(first_child.key());
return result;
}
uint32_t DataInnerNode::numChildren() const {
return node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
return const_cast(const_cast(this)->ChildrenBegin());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() const {
return node().DataBegin();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
return const_cast(const_cast(this)->ChildrenEnd());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() const {
return ChildrenBegin() + node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::LastChild() {
return const_cast(const_cast(this)->LastChild());
}
const DataInnerNode::ChildEntry *DataInnerNode::LastChild() const {
return getChild(numChildren()-1);
}
DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) {
return const_cast(const_cast(this)->getChild(index));
}
const DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) const {
ASSERT(index < numChildren(), "Accessing child out of range");
return ChildrenBegin()+index;
}
void DataInnerNode::addChild(const DataNode &child) {
ASSERT(numChildren() < maxStoreableChildren(), "Adding more children than we can store");
ASSERT(child.depth() == depth()-1, "The child that should be added has wrong depth");
node().setSize(node().Size()+1);
LastChild()->setKey(child.key());
}
void DataInnerNode::removeLastChild() {
ASSERT(node().Size() > 1, "There is no child to remove");
node().setSize(node().Size()-1);
}
uint32_t DataInnerNode::maxStoreableChildren() const {
return node().layout().maxChildrenPerInnerNode();
}
}
}
}
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h 0000664 0000000 0000000 00000002170 13235746211 0026346 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
#include "DataNode.h"
#include "DataInnerNode_ChildEntry.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataInnerNode final: public DataNode {
public:
static cpputils::unique_ref InitializeNewNode(cpputils::unique_ref block, const DataNode &first_child_key);
DataInnerNode(DataNodeView block);
~DataInnerNode();
using ChildEntry = DataInnerNode_ChildEntry;
uint32_t maxStoreableChildren() const;
ChildEntry *getChild(unsigned int index);
const ChildEntry *getChild(unsigned int index) const;
uint32_t numChildren() const;
void addChild(const DataNode &child_key);
void removeLastChild();
ChildEntry *LastChild();
const ChildEntry *LastChild() const;
private:
ChildEntry *ChildrenBegin();
ChildEntry *ChildrenEnd();
const ChildEntry *ChildrenBegin() const;
const ChildEntry *ChildrenEnd() const;
DISALLOW_COPY_AND_ASSIGN(DataInnerNode);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h 0000664 0000000 0000000 00000001240 13235746211 0030470 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
#include
namespace blobstore{
namespace onblocks{
namespace datanodestore{
struct DataInnerNode_ChildEntry final {
public:
blockstore::Key key() const {
return blockstore::Key::FromBinary(_keydata);
}
private:
void setKey(const blockstore::Key &key) {
key.ToBinary(_keydata);
}
friend class DataInnerNode;
uint8_t _keydata[blockstore::Key::BINARY_LENGTH];
DISALLOW_COPY_AND_ASSIGN(DataInnerNode_ChildEntry);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp 0000664 0000000 0000000 00000004610 13235746211 0026476 0 ustar 00root root 0000000 0000000 #include "DataLeafNode.h"
#include "DataInnerNode.h"
#include
using blockstore::Block;
using cpputils::Data;
using blockstore::Key;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataLeafNode::DataLeafNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(node().Depth() == 0, "Leaf node must have depth 0. Is it an inner node instead?");
ASSERT(numBytes() <= maxStoreableBytes(), "Leaf says it stores more bytes than it has space for");
if (node().FormatVersion() != FORMAT_VERSION_HEADER) {
throw std::runtime_error("This node format is not supported. Was it created with a newer version of CryFS?");
}
}
DataLeafNode::~DataLeafNode() {
}
unique_ref DataLeafNode::InitializeNewNode(unique_ref block) {
DataNodeView node(std::move(block));
node.setFormatVersion(DataNode::FORMAT_VERSION_HEADER);
node.setDepth(0);
node.setSize(0);
//fillDataWithZeroes(); not needed, because a newly created block will be zeroed out. DataLeafNodeTest.SpaceIsZeroFilledWhenGrowing ensures this.
return make_unique_ref(std::move(node));
}
void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
std::memcpy(target, (uint8_t*)node().data() + offset, size);
}
void DataLeafNode::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Write out of valid area"); // Also check offset, because the addition could lead to overflows
node().write(source, offset, size);
}
uint32_t DataLeafNode::numBytes() const {
return node().Size();
}
void DataLeafNode::resize(uint32_t new_size) {
ASSERT(new_size <= maxStoreableBytes(), "Trying to resize to a size larger than the maximal size");
uint32_t old_size = node().Size();
if (new_size < old_size) {
fillDataWithZeroesFromTo(new_size, old_size);
}
node().setSize(new_size);
}
void DataLeafNode::fillDataWithZeroesFromTo(off_t begin, off_t end) {
Data ZEROES(end-begin);
ZEROES.FillWithZeroes();
node().write(ZEROES.data(), begin, end-begin);
}
uint64_t DataLeafNode::maxStoreableBytes() const {
return node().layout().maxBytesPerLeaf();
}
}
}
}
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h 0000664 0000000 0000000 00000001730 13235746211 0026143 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
#include "DataNode.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataInnerNode;
class DataLeafNode final: public DataNode {
public:
static cpputils::unique_ref InitializeNewNode(cpputils::unique_ref block);
DataLeafNode(DataNodeView block);
~DataLeafNode();
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxStoreableBytes() const;
void read(void *target, uint64_t offset, uint64_t size) const;
void write(const void *source, uint64_t offset, uint64_t size);
uint32_t numBytes() const;
void resize(uint32_t size);
private:
void fillDataWithZeroesFromTo(off_t begin, off_t end);
DISALLOW_COPY_AND_ASSIGN(DataLeafNode);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp 0000664 0000000 0000000 00000002164 13235746211 0025710 0 ustar 00root root 0000000 0000000 #include "DataInnerNode.h"
#include "DataLeafNode.h"
#include "DataNode.h"
#include "DataNodeStore.h"
#include
using blockstore::Block;
using blockstore::Key;
using std::runtime_error;
using cpputils::unique_ref;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
constexpr uint16_t DataNode::FORMAT_VERSION_HEADER;
DataNode::DataNode(DataNodeView node)
: _node(std::move(node)) {
}
DataNode::~DataNode() {
}
DataNodeView &DataNode::node() {
return const_cast(const_cast(this)->node());
}
const DataNodeView &DataNode::node() const {
return _node;
}
const Key &DataNode::key() const {
return _node.key();
}
uint8_t DataNode::depth() const {
return _node.Depth();
}
unique_ref DataNode::convertToNewInnerNode(unique_ref node, const DataNode &first_child) {
Key key = node->key();
auto block = node->_node.releaseBlock();
blockstore::utils::fillWithZeroes(block.get());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
}
void DataNode::flush() const {
_node.flush();
}
}
}
}
src/blobstore/implementations/onblocks/datanodestore/DataNode.h 0000664 0000000 0000000 00000001723 13235746211 0025355 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
#include "DataNodeView.h"
#include
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataInnerNode;
class DataNode {
public:
virtual ~DataNode();
const blockstore::Key &key() const;
uint8_t depth() const;
static cpputils::unique_ref convertToNewInnerNode(cpputils::unique_ref node, const DataNode &first_child);
void flush() const;
protected:
// The FORMAT_VERSION_HEADER is used to allow future versions to have compatibility.
static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
DataNode(DataNodeView block);
DataNodeView &node();
const DataNodeView &node() const;
friend class DataNodeStore;
private:
DataNodeView _node;
DISALLOW_COPY_AND_ASSIGN(DataNode);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp 0000664 0000000 0000000 00000010765 13235746211 0026733 0 ustar 00root root 0000000 0000000 #include "DataInnerNode.h"
#include "DataLeafNode.h"
#include "DataNodeStore.h"
#include
#include
#include
#include
using blockstore::BlockStore;
using blockstore::Block;
using blockstore::Key;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using std::runtime_error;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataNodeStore::DataNodeStore(unique_ref blockstore, uint64_t physicalBlocksizeBytes)
: _blockstore(std::move(blockstore)), _layout(_blockstore->blockSizeFromPhysicalBlockSize(physicalBlocksizeBytes)) {
}
DataNodeStore::~DataNodeStore() {
}
unique_ref DataNodeStore::load(unique_ref block) {
ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
DataNodeView node(std::move(block));
if (node.Depth() == 0) {
return make_unique_ref(std::move(node));
} else if (node.Depth() <= MAX_DEPTH) {
return make_unique_ref(std::move(node));
} else {
throw runtime_error("Tree is to deep. Data corruption?");
}
}
unique_ref DataNodeStore::createNewInnerNode(const DataNode &first_child) {
ASSERT(first_child.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
}
unique_ref DataNodeStore::createNewLeafNode() {
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataLeafNode::InitializeNewNode(std::move(block));
}
optional> DataNodeStore::load(const Key &key) {
auto block = _blockstore->load(key);
if (block == none) {
return none;
} else {
return load(std::move(*block));
}
}
unique_ref DataNodeStore::createNewNodeAsCopyFrom(const DataNode &source) {
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
auto newBlock = blockstore::utils::copyToNewBlock(_blockstore.get(), source.node().block());
return load(std::move(newBlock));
}
unique_ref DataNodeStore::overwriteNodeWith(unique_ref target, const DataNode &source) {
ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
Key key = target->key();
{
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
}
auto loaded = load(key);
ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
return std::move(*loaded);
}
void DataNodeStore::remove(unique_ref node) {
auto block = node->node().releaseBlock();
cpputils::destruct(std::move(node)); // Call destructor
_blockstore->remove(std::move(block));
}
uint64_t DataNodeStore::numNodes() const {
return _blockstore->numBlocks();
}
uint64_t DataNodeStore::estimateSpaceForNumNodesLeft() const {
return _blockstore->estimateNumFreeBytes() / _layout.blocksizeBytes();
}
uint64_t DataNodeStore::virtualBlocksizeBytes() const {
return _layout.blocksizeBytes();
}
void DataNodeStore::removeSubtree(unique_ref node) {
//TODO Make this faster by not loading the leaves but just deleting them. Can be recognized, because of the depth of their parents.
DataInnerNode *inner = dynamic_cast(node.get());
if (inner != nullptr) {
for (uint32_t i = 0; i < inner->numChildren(); ++i) {
auto child = load(inner->getChild(i)->key());
ASSERT(child != none, "Couldn't load child node");
removeSubtree(std::move(*child));
}
}
remove(std::move(node));
}
DataNodeLayout DataNodeStore::layout() const {
return _layout;
}
}
}
}
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h 0000664 0000000 0000000 00000003352 13235746211 0026372 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
#include
#include
#include "DataNodeView.h"
#include
namespace blockstore{
class Block;
class BlockStore;
}
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNode;
class DataLeafNode;
class DataInnerNode;
class DataNodeStore final {
public:
DataNodeStore(cpputils::unique_ref blockstore, uint64_t physicalBlocksizeBytes);
~DataNodeStore();
static constexpr uint8_t MAX_DEPTH = 10;
DataNodeLayout layout() const;
boost::optional> load(const blockstore::Key &key);
cpputils::unique_ref createNewLeafNode();
cpputils::unique_ref createNewInnerNode(const DataNode &first_child);
cpputils::unique_ref createNewNodeAsCopyFrom(const DataNode &source);
cpputils::unique_ref overwriteNodeWith(cpputils::unique_ref target, const DataNode &source);
void remove(cpputils::unique_ref node);
void removeSubtree(cpputils::unique_ref node);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t virtualBlocksizeBytes() const;
uint64_t numNodes() const;
uint64_t estimateSpaceForNumNodesLeft() const;
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
private:
cpputils::unique_ref load(cpputils::unique_ref block);
cpputils::unique_ref _blockstore;
const DataNodeLayout _layout;
DISALLOW_COPY_AND_ASSIGN(DataNodeStore);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h 0000664 0000000 0000000 00000010054 13235746211 0026205 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
#include
#include "../BlobStoreOnBlocks.h"
#include "DataInnerNode_ChildEntry.h"
#include
#include
#include
#include
namespace blobstore {
namespace onblocks {
namespace datanodestore {
//TODO Move DataNodeLayout into own file
class DataNodeLayout final {
public:
constexpr DataNodeLayout(uint64_t blocksizeBytes)
:_blocksizeBytes(
(HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes)
? blocksizeBytes
: throw std::logic_error("Blocksize too small, not enough space to store two children in an inner node")) {
}
//Total size of the header
static constexpr uint32_t HEADERSIZE_BYTES = 8;
//Where in the header is the format version field (used to allow compatibility with future versions of CryFS)
static constexpr uint32_t FORMAT_VERSION_OFFSET_BYTES = 0; //format version uses 2 bytes
//Where in the header is the depth field
static constexpr uint32_t DEPTH_OFFSET_BYTES = 3; // depth uses 1 byte
//Where in the header is the size field (for inner nodes: number of children, for leafs: content data size)
static constexpr uint32_t SIZE_OFFSET_BYTES = 4; // size uses 4 bytes
//Size of a block (header + data region)
constexpr uint64_t blocksizeBytes() const {
return _blocksizeBytes;
}
//Number of bytes in the data region of a node
constexpr uint64_t datasizeBytes() const {
return _blocksizeBytes - HEADERSIZE_BYTES;
}
//Maximum number of children an inner node can store
constexpr uint64_t maxChildrenPerInnerNode() const {
return datasizeBytes() / sizeof(DataInnerNode_ChildEntry);
}
//Maximum number of bytes a leaf can store
constexpr uint64_t maxBytesPerLeaf() const {
return datasizeBytes();
}
private:
uint32_t _blocksizeBytes;
};
class DataNodeView final {
public:
DataNodeView(cpputils::unique_ref block): _block(std::move(block)) {
}
~DataNodeView() {}
DataNodeView(DataNodeView &&rhs) = default;
uint16_t FormatVersion() const {
return *((uint8_t*)_block->data()+DataNodeLayout::FORMAT_VERSION_OFFSET_BYTES);
}
void setFormatVersion(uint16_t value) {
_block->write(&value, DataNodeLayout::FORMAT_VERSION_OFFSET_BYTES, sizeof(value));
}
uint8_t Depth() const {
return *((uint8_t*)_block->data()+DataNodeLayout::DEPTH_OFFSET_BYTES);
}
void setDepth(uint8_t value) {
_block->write(&value, DataNodeLayout::DEPTH_OFFSET_BYTES, sizeof(value));
}
uint32_t Size() const {
return *(uint32_t*)((uint8_t*)_block->data()+DataNodeLayout::SIZE_OFFSET_BYTES);
}
void setSize(uint32_t value) {
_block->write(&value, DataNodeLayout::SIZE_OFFSET_BYTES, sizeof(value));
}
const void *data() const {
return (uint8_t*)_block->data() + DataNodeLayout::HEADERSIZE_BYTES;
}
void write(const void *source, uint64_t offset, uint64_t size) {
_block->write(source, offset + DataNodeLayout::HEADERSIZE_BYTES, size);
}
template
const Entry *DataBegin() const {
return GetOffset();
}
template
const Entry *DataEnd() const {
const unsigned int NUM_ENTRIES = layout().datasizeBytes() / sizeof(Entry);
return DataBegin() + NUM_ENTRIES;
}
DataNodeLayout layout() const {
return DataNodeLayout(_block->size());
}
cpputils::unique_ref releaseBlock() {
return std::move(_block);
}
const blockstore::Block &block() const {
return *_block;
}
const blockstore::Key &key() const {
return _block->key();
}
void flush() const {
_block->flush();
}
private:
template
const Type *GetOffset() const {
return (Type*)(((const int8_t*)_block->data())+offset);
}
cpputils::unique_ref _block;
DISALLOW_COPY_AND_ASSIGN(DataNodeView);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datatreestore/ 0000775 0000000 0000000 00000000000 13235746211 0023534 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp 0000664 0000000 0000000 00000032607 13235746211 0025741 0 ustar 00root root 0000000 0000000 #include "DataTree.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataInnerNode.h"
#include "../datanodestore/DataLeafNode.h"
#include "../utils/Math.h"
#include "impl/algorithms.h"
#include
#include
#include
#include
using blockstore::Key;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using std::dynamic_pointer_cast;
using std::function;
using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
using boost::none;
using std::vector;
using cpputils::dynamic_pointer_move;
using cpputils::optional_ownership_ptr;
using cpputils::WithOwnership;
using cpputils::WithoutOwnership;
using cpputils::unique_ref;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTree::DataTree(DataNodeStore *nodeStore, unique_ref rootNode)
: _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)) {
}
DataTree::~DataTree() {
}
void DataTree::removeLastDataLeaf() {
auto deletePosOrNull = algorithms::GetLowestRightBorderNodeWithMoreThanOneChildOrNull(_nodeStore, _rootNode.get());
ASSERT(deletePosOrNull.get() != nullptr, "Tree has only one leaf, can't shrink it.");
deleteLastChildSubtree(deletePosOrNull.get());
ifRootHasOnlyOneChildReplaceRootWithItsChild();
}
void DataTree::ifRootHasOnlyOneChildReplaceRootWithItsChild() {
DataInnerNode *rootNode = dynamic_cast(_rootNode.get());
ASSERT(rootNode != nullptr, "RootNode is not an inner node");
if (rootNode->numChildren() == 1) {
auto child = _nodeStore->load(rootNode->getChild(0)->key());
ASSERT(child != none, "Couldn't load first child of root node");
_rootNode = _nodeStore->overwriteNodeWith(std::move(_rootNode), **child);
_nodeStore->remove(std::move(*child));
}
}
void DataTree::deleteLastChildSubtree(DataInnerNode *node) {
auto lastChild = _nodeStore->load(node->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
_nodeStore->removeSubtree(std::move(*lastChild));
node->removeLastChild();
}
unique_ref DataTree::addDataLeaf() {
auto insertPosOrNull = algorithms::GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(_nodeStore, _rootNode.get());
if (insertPosOrNull) {
return addDataLeafAt(insertPosOrNull.get());
} else {
return addDataLeafToFullTree();
}
}
unique_ref DataTree::addDataLeafAt(DataInnerNode *insertPos) {
auto new_leaf = _nodeStore->createNewLeafNode();
auto chain = createChainOfInnerNodes(insertPos->depth()-1, new_leaf.get());
insertPos->addChild(*chain);
return new_leaf;
}
optional_ownership_ptr DataTree::createChainOfInnerNodes(unsigned int num, DataNode *child) {
//TODO This function is implemented twice, once with optional_ownership_ptr, once with unique_ref. Redundancy!
optional_ownership_ptr chain = cpputils::WithoutOwnership(child);
for(unsigned int i=0; icreateNewInnerNode(*chain);
chain = cpputils::WithOwnership(std::move(newnode));
}
return chain;
}
unique_ref DataTree::createChainOfInnerNodes(unsigned int num, unique_ref child) {
unique_ref chain = std::move(child);
for(unsigned int i=0; icreateNewInnerNode(*chain);
}
return chain;
}
DataInnerNode* DataTree::increaseTreeDepth(unsigned int levels) {
ASSERT(levels >= 1, "Parameter out of bounds: tried to increase tree depth by zero.");
auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*_rootNode);
auto chain = createChainOfInnerNodes(levels-1, copyOfOldRoot.get());
auto newRootNode = DataNode::convertToNewInnerNode(std::move(_rootNode), *chain);
DataInnerNode *result = newRootNode.get();
_rootNode = std::move(newRootNode);
return result;
}
unique_ref DataTree::addDataLeafToFullTree() {
DataInnerNode *rootNode = increaseTreeDepth(1);
auto newLeaf = addDataLeafAt(rootNode);
return newLeaf;
}
const Key &DataTree::key() const {
return _rootNode->key();
}
void DataTree::flush() const {
// By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed
unique_lock lock(_mutex);
// We also have to flush the root node
_rootNode->flush();
}
unique_ref DataTree::releaseRootNode() {
return std::move(_rootNode);
}
//TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
uint32_t DataTree::numLeaves() const {
shared_lock lock(_mutex);
return _numLeaves(*_rootNode);
}
uint32_t DataTree::_numLeaves(const DataNode &node) const {
const DataLeafNode *leaf = dynamic_cast(&node);
if (leaf != nullptr) {
return 1;
}
const DataInnerNode &inner = dynamic_cast(node);
uint64_t numLeavesInLeftChildren = (uint64_t)(inner.numChildren()-1) * leavesPerFullChild(inner);
auto lastChild = _nodeStore->load(inner.LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numLeavesInRightChild = _numLeaves(**lastChild);
return numLeavesInLeftChildren + numLeavesInRightChild;
}
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function func) {
//TODO Can we traverse in parallel?
unique_lock lock(_mutex); //TODO Only lock when resizing. Otherwise parallel read/write to a blob is not possible!
ASSERT(beginIndex <= endIndex, "Invalid parameters");
if (0 == endIndex) {
// In this case the utils::ceilLog(_, endIndex) below would fail
return;
}
uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)endIndex);
uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
if (_rootNode->depth() < neededTreeDepth) {
//TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
increaseTreeDepth(neededTreeDepth - _rootNode->depth());
}
if (numLeaves <= beginIndex) {
//TODO Test cases with numLeaves < / >= beginIndex
// There is a gap between the current size and the begin of the traversal
return _traverseLeaves(_rootNode.get(), 0, numLeaves-1, endIndex, [beginIndex, numLeaves, &func, this](DataLeafNode* node, uint32_t index) {
if (index >= beginIndex) {
func(node, index);
} else if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
});
} else if (numLeaves < endIndex) {
// We are starting traversal in the valid region, but traverse until after it (we grow new leaves)
return _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, [numLeaves, &func, this] (DataLeafNode *node, uint32_t index) {
if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
func(node, index);
});
} else {
//We are traversing entirely inside the valid region
_traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, func);
}
}
void DataTree::_traverseLeaves(DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, function func) {
DataLeafNode *leaf = dynamic_cast(root);
if (leaf != nullptr) {
ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
if (beginIndex == 0 && endIndex == 1) {
func(leaf, leafOffset);
}
return;
}
DataInnerNode *inner = dynamic_cast(root);
uint32_t leavesPerChild = leavesPerFullChild(*inner);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
vector> children = getOrCreateChildren(inner, beginChild, endChild);
for (uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto child = std::move(children[childIndex-beginChild]);
_traverseLeaves(child.get(), leafOffset + childOffset, localBeginIndex, localEndIndex, func);
}
}
vector> DataTree::getOrCreateChildren(DataInnerNode *node, uint32_t begin, uint32_t end) {
vector> children;
children.reserve(end-begin);
for (uint32_t childIndex = begin; childIndex < std::min(node->numChildren(), end); ++childIndex) {
auto child = _nodeStore->load(node->getChild(childIndex)->key());
ASSERT(child != none, "Couldn't load child node");
children.emplace_back(std::move(*child));
}
for (uint32_t childIndex = node->numChildren(); childIndex < end; ++childIndex) {
//TODO This creates each child with one chain to one leaf only, and then on the next lower level it
// has to create the children for the child. Would be faster to directly create full trees if necessary.
children.emplace_back(addChildTo(node));
}
ASSERT(children.size() == end-begin, "Number of children in the result is wrong");
return children;
}
unique_ref DataTree::addChildTo(DataInnerNode *node) {
auto new_leaf = _nodeStore->createNewLeafNode();
new_leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
auto chain = createChainOfInnerNodes(node->depth()-1, std::move(new_leaf));
node->addChild(*chain);
return chain;
}
uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)root.depth()-1);
}
uint64_t DataTree::numStoredBytes() const {
shared_lock lock(_mutex);
return _numStoredBytes();
}
uint64_t DataTree::_numStoredBytes() const {
return _numStoredBytes(*_rootNode);
}
uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
const DataLeafNode *leaf = dynamic_cast(&root);
if (leaf != nullptr) {
return leaf->numBytes();
}
const DataInnerNode &inner = dynamic_cast(root);
uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
auto lastChild = _nodeStore->load(inner.LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
return numBytesInLeftChildren + numBytesInRightChild;
}
void DataTree::resizeNumBytes(uint64_t newNumBytes) {
//TODO Can we resize in parallel? Especially creating new blocks (i.e. encrypting them) is expensive and should be done in parallel.
boost::upgrade_lock lock(_mutex);
{
boost::upgrade_to_unique_lock exclusiveLock(lock);
//TODO Faster implementation possible (no addDataLeaf()/removeLastDataLeaf() in a loop, but directly resizing)
LastLeaf(_rootNode.get())->resize(_nodeStore->layout().maxBytesPerLeaf());
uint64_t currentNumBytes = _numStoredBytes();
ASSERT(currentNumBytes % _nodeStore->layout().maxBytesPerLeaf() == 0, "The last leaf is not a max data leaf, although we just resized it to be one.");
uint32_t currentNumLeaves = currentNumBytes / _nodeStore->layout().maxBytesPerLeaf();
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
for(uint32_t i = currentNumLeaves; i < newNumLeaves; ++i) {
addDataLeaf()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
for(uint32_t i = currentNumLeaves; i > newNumLeaves; --i) {
removeLastDataLeaf();
}
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1)*_nodeStore->layout().maxBytesPerLeaf();
LastLeaf(_rootNode.get())->resize(newLastLeafSize);
}
ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
}
optional_ownership_ptr DataTree::LastLeaf(DataNode *root) {
DataLeafNode *leaf = dynamic_cast(root);
if (leaf != nullptr) {
return WithoutOwnership(leaf);
}
DataInnerNode *inner = dynamic_cast(root);
auto lastChild = _nodeStore->load(inner->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
return WithOwnership(LastLeaf(std::move(*lastChild)));
}
unique_ref DataTree::LastLeaf(unique_ref root) {
auto leaf = dynamic_pointer_move(root);
if (leaf != none) {
return std::move(*leaf);
}
auto inner = dynamic_pointer_move(root);
ASSERT(inner != none, "Root node is neither a leaf nor an inner node");
auto child = _nodeStore->load((*inner)->LastChild()->key());
ASSERT(child != none, "Couldn't load last child");
return LastLeaf(std::move(*child));
}
uint64_t DataTree::maxBytesPerLeaf() const {
return _nodeStore->layout().maxBytesPerLeaf();
}
}
}
}
src/blobstore/implementations/onblocks/datatreestore/DataTree.h 0000664 0000000 0000000 00000006414 13235746211 0025403 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
#include
#include
#include
#include "../datanodestore/DataNodeView.h"
//TODO Replace with C++14 once std::shared_mutex is supported
#include
#include
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataInnerNode;
class DataLeafNode;
class DataNode;
}
namespace datatreestore {
//TODO It is strange that DataLeafNode is still part in the public interface of DataTree. This should be separated somehow.
class DataTree final {
public:
DataTree(datanodestore::DataNodeStore *nodeStore, cpputils::unique_ref rootNode);
~DataTree();
const blockstore::Key &key() const;
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxBytesPerLeaf() const;
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function func);
void resizeNumBytes(uint64_t newNumBytes);
uint32_t numLeaves() const;
uint64_t numStoredBytes() const;
void flush() const;
private:
mutable boost::shared_mutex _mutex;
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref _rootNode;
cpputils::unique_ref addDataLeaf();
void removeLastDataLeaf();
cpputils::unique_ref releaseRootNode();
friend class DataTreeStore;
cpputils::unique_ref addDataLeafAt(datanodestore::DataInnerNode *insertPos);
cpputils::optional_ownership_ptr createChainOfInnerNodes(unsigned int num, datanodestore::DataNode *child);
cpputils::unique_ref createChainOfInnerNodes(unsigned int num, cpputils::unique_ref child);
cpputils::unique_ref addDataLeafToFullTree();
void deleteLastChildSubtree(datanodestore::DataInnerNode *node);
void ifRootHasOnlyOneChildReplaceRootWithItsChild();
//TODO Use underscore for private methods
void _traverseLeaves(datanodestore::DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, std::function func);
uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
uint64_t _numStoredBytes() const;
uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
uint32_t _numLeaves(const datanodestore::DataNode &node) const;
cpputils::optional_ownership_ptr LastLeaf(datanodestore::DataNode *root);
cpputils::unique_ref LastLeaf(cpputils::unique_ref root);
datanodestore::DataInnerNode* increaseTreeDepth(unsigned int levels);
std::vector> getOrCreateChildren(datanodestore::DataInnerNode *node, uint32_t begin, uint32_t end);
cpputils::unique_ref addChildTo(datanodestore::DataInnerNode *node);
DISALLOW_COPY_AND_ASSIGN(DataTree);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp 0000664 0000000 0000000 00000002245 13235746211 0026751 0 ustar 00root root 0000000 0000000 #include "DataTreeStore.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
#include "DataTree.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTreeStore::DataTreeStore(unique_ref nodeStore)
: _nodeStore(std::move(nodeStore)) {
}
DataTreeStore::~DataTreeStore() {
}
optional> DataTreeStore::load(const blockstore::Key &key) {
auto node = _nodeStore->load(key);
if (node == none) {
return none;
}
return make_unique_ref(_nodeStore.get(), std::move(*node));
}
unique_ref DataTreeStore::createNewTree() {
auto newleaf = _nodeStore->createNewLeafNode();
return make_unique_ref(_nodeStore.get(), std::move(newleaf));
}
void DataTreeStore::remove(unique_ref tree) {
auto root = tree->releaseRootNode();
cpputils::destruct(std::move(tree)); // Destruct tree
_nodeStore->removeSubtree(std::move(root));
}
}
}
}
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h 0000664 0000000 0000000 00000002644 13235746211 0026421 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
#include
#include
#include
#include
#include
#include "../datanodestore/DataNodeStore.h"
namespace blobstore {
namespace onblocks {
namespace datatreestore {
class DataTree;
class DataTreeStore final {
public:
DataTreeStore(cpputils::unique_ref nodeStore);
~DataTreeStore();
boost::optional> load(const blockstore::Key &key);
cpputils::unique_ref createNewTree();
void remove(cpputils::unique_ref tree);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t virtualBlocksizeBytes() const;
uint64_t numNodes() const;
uint64_t estimateSpaceForNumNodesLeft() const;
private:
cpputils::unique_ref _nodeStore;
DISALLOW_COPY_AND_ASSIGN(DataTreeStore);
};
inline uint64_t DataTreeStore::numNodes() const {
return _nodeStore->numNodes();
}
inline uint64_t DataTreeStore::estimateSpaceForNumNodesLeft() const {
return _nodeStore->estimateSpaceForNumNodesLeft();
}
inline uint64_t DataTreeStore::virtualBlocksizeBytes() const {
return _nodeStore->virtualBlocksizeBytes();
}
}
}
}
#endif
src/blobstore/implementations/onblocks/datatreestore/impl/ 0000775 0000000 0000000 00000000000 13235746211 0024475 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp 0000664 0000000 0000000 00000005432 13235746211 0027356 0 ustar 00root root 0000000 0000000 #include "algorithms.h"
#include
#include
#include "../../datanodestore/DataInnerNode.h"
#include "../../datanodestore/DataNodeStore.h"
#include
using std::function;
using cpputils::optional_ownership_ptr;
using cpputils::dynamic_pointer_move;
using cpputils::unique_ref;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blockstore::Key;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
namespace algorithms {
optional> getLastChildAsInnerNode(DataNodeStore *nodeStore, const DataInnerNode &node) {
Key key = node.LastChild()->key();
auto lastChild = nodeStore->load(key);
ASSERT(lastChild != none, "Couldn't load last child");
return dynamic_pointer_move(*lastChild);
}
//Returns the lowest right border node meeting the condition specified (exclusive the leaf).
//Returns nullptr, if no inner right border node meets the condition.
optional_ownership_ptr GetLowestInnerRightBorderNodeWithConditionOrNull(DataNodeStore *nodeStore, datanodestore::DataNode *rootNode, function condition) {
optional_ownership_ptr currentNode = cpputils::WithoutOwnership(dynamic_cast(rootNode));
optional_ownership_ptr result = cpputils::null();
for (unsigned int i=0; i < rootNode->depth(); ++i) {
//TODO This unnecessarily loads the leaf node in the last loop run
auto lastChild = getLastChildAsInnerNode(nodeStore, *currentNode);
if (condition(*currentNode)) {
result = std::move(currentNode);
}
ASSERT(lastChild != none || static_cast(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
if (lastChild != none) {
currentNode = cpputils::WithOwnership(std::move(*lastChild));
}
}
return result;
}
optional_ownership_ptr GetLowestRightBorderNodeWithMoreThanOneChildOrNull(DataNodeStore *nodeStore, DataNode *rootNode) {
return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
return node.numChildren() > 1;
});
}
optional_ownership_ptr GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode) {
return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
return node.numChildren() < node.maxStoreableChildren();
});
}
}
}
}
}
src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.h 0000664 0000000 0000000 00000002301 13235746211 0027013 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
#include
namespace blobstore {
namespace onblocks {
namespace datanodestore{
class DataNode;
class DataInnerNode;
class DataNodeStore;
}
namespace datatreestore {
namespace algorithms {
//Returns the lowest right border node with at least two children.
//Returns nullptr, if all right border nodes have only one child (since the root is a right border node, this means that the whole tree has exactly one leaf)
cpputils::optional_ownership_ptr GetLowestRightBorderNodeWithMoreThanOneChildOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
//Returns the lowest right border node with less than k children (not considering leaves).
//Returns nullptr, if all right border nodes have k children (the tree is full)
cpputils::optional_ownership_ptr GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
}
}
}
}
#endif
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ 0000775 0000000 0000000 00000000000 13235746211 0026433 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp 0000664 0000000 0000000 00000000031 13235746211 0031257 0 ustar 00root root 0000000 0000000 #include "DataTreeRef.h"
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h 0000664 0000000 0000000 00000002545 13235746211 0030740 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
#include
#include "../datatreestore/DataTree.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class DataTreeRef final: public parallelaccessstore::ParallelAccessStore::ResourceRefBase {
public:
DataTreeRef(datatreestore::DataTree *baseTree): _baseTree(baseTree) {}
const blockstore::Key &key() const {
return _baseTree->key();
}
uint64_t maxBytesPerLeaf() const {
return _baseTree->maxBytesPerLeaf();
}
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function func) {
return _baseTree->traverseLeaves(beginIndex, endIndex, func);
}
uint32_t numLeaves() const {
return _baseTree->numLeaves();
}
void resizeNumBytes(uint64_t newNumBytes) {
return _baseTree->resizeNumBytes(newNumBytes);
}
uint64_t numStoredBytes() const {
return _baseTree->numStoredBytes();
}
void flush() {
return _baseTree->flush();
}
private:
datatreestore::DataTree *_baseTree;
DISALLOW_COPY_AND_ASSIGN(DataTreeRef);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp 0000664 0000000 0000000 00000002637 13235746211 0034454 0 ustar 00root root 0000000 0000000 #include "DataTreeRef.h"
#include "ParallelAccessDataTreeStore.h"
#include "ParallelAccessDataTreeStoreAdapter.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using blobstore::onblocks::datatreestore::DataTreeStore;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
using datatreestore::DataTreeStore;
using datatreestore::DataTree;
namespace parallelaccessdatatreestore {
//TODO Here and for other stores (DataTreeStore, ...): Make small functions inline
ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref dataTreeStore)
: _dataTreeStore(std::move(dataTreeStore)), _parallelAccessStore(make_unique_ref(_dataTreeStore.get())) {
}
ParallelAccessDataTreeStore::~ParallelAccessDataTreeStore() {
}
optional> ParallelAccessDataTreeStore::load(const blockstore::Key &key) {
return _parallelAccessStore.load(key);
}
unique_ref ParallelAccessDataTreeStore::createNewTree() {
auto dataTree = _dataTreeStore->createNewTree();
Key key = dataTree->key();
return _parallelAccessStore.add(key, std::move(dataTree));
}
void ParallelAccessDataTreeStore::remove(unique_ref tree) {
Key key = tree->key();
return _parallelAccessStore.remove(key, std::move(tree));
}
}
}
}
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h 0000664 0000000 0000000 00000003363 13235746211 0034116 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
#include
#include
#include
#include
#include "../datatreestore/DataTreeStore.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class DataTreeRef;
//TODO Test CachingDataTreeStore
class ParallelAccessDataTreeStore final {
public:
ParallelAccessDataTreeStore(cpputils::unique_ref dataTreeStore);
~ParallelAccessDataTreeStore();
boost::optional> load(const blockstore::Key &key);
cpputils::unique_ref createNewTree();
void remove(cpputils::unique_ref tree);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t virtualBlocksizeBytes() const;
uint64_t numNodes() const;
uint64_t estimateSpaceForNumNodesLeft() const;
private:
cpputils::unique_ref _dataTreeStore;
parallelaccessstore::ParallelAccessStore _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
};
inline uint64_t ParallelAccessDataTreeStore::virtualBlocksizeBytes() const {
return _dataTreeStore->virtualBlocksizeBytes();
}
inline uint64_t ParallelAccessDataTreeStore::numNodes() const {
return _dataTreeStore->numNodes();
}
inline uint64_t ParallelAccessDataTreeStore::estimateSpaceForNumNodesLeft() const {
return _dataTreeStore->estimateSpaceForNumNodesLeft();
}
}
}
}
#endif
ParallelAccessDataTreeStoreAdapter.cpp 0000664 0000000 0000000 00000000060 13235746211 0035662 0 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/parallelaccessdatatreestore #include "ParallelAccessDataTreeStoreAdapter.h"
ParallelAccessDataTreeStoreAdapter.h 0000664 0000000 0000000 00000002364 13235746211 0035340 0 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/parallelaccessdatatreestore #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
#include
#include
#include "../datatreestore/DataTreeStore.h"
#include "../datatreestore/DataTree.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore {
public:
ParallelAccessDataTreeStoreAdapter(datatreestore::DataTreeStore *baseDataTreeStore)
:_baseDataTreeStore(std::move(baseDataTreeStore)) {
}
boost::optional> loadFromBaseStore(const blockstore::Key &key) override {
return _baseDataTreeStore->load(key);
}
void removeFromBaseStore(cpputils::unique_ref dataTree) override {
return _baseDataTreeStore->remove(std::move(dataTree));
}
private:
datatreestore::DataTreeStore *_baseDataTreeStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStoreAdapter);
};
}
}
}
#endif
src/blobstore/implementations/onblocks/utils/ 0000775 0000000 0000000 00000000000 13235746211 0022026 5 ustar 00root root 0000000 0000000 src/blobstore/implementations/onblocks/utils/Math.cpp 0000664 0000000 0000000 00000000022 13235746211 0023415 0 ustar 00root root 0000000 0000000 #include "Math.h"
src/blobstore/implementations/onblocks/utils/Math.h 0000664 0000000 0000000 00000001715 13235746211 0023074 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
#include
#include
namespace blobstore {
namespace onblocks {
namespace utils {
template
inline INT_TYPE intPow(INT_TYPE base, INT_TYPE exponent) {
INT_TYPE result = 1;
for(INT_TYPE i = 0; i < exponent; ++i) {
result *= base;
}
return result;
}
template
inline INT_TYPE ceilDivision(INT_TYPE dividend, INT_TYPE divisor) {
return (dividend + divisor - 1)/divisor;
}
template
inline INT_TYPE maxZeroSubtraction(INT_TYPE minuend, INT_TYPE subtrahend) {
if (minuend < subtrahend) {
return 0u;
}
return minuend-subtrahend;
}
template
inline INT_TYPE ceilLog(INT_TYPE base, INT_TYPE value) {
return std::ceil((long double)std::log(value)/(long double)std::log(base));
}
}
}
}
#endif
src/blobstore/interface/ 0000775 0000000 0000000 00000000000 13235746211 0015604 5 ustar 00root root 0000000 0000000 src/blobstore/interface/Blob.h 0000664 0000000 0000000 00000001442 13235746211 0016634 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
#define MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
#include
#include
#include
#include
namespace blobstore {
class Blob {
public:
virtual ~Blob() {}
//TODO Use own Key class for blobstore
virtual const blockstore::Key &key() const = 0;
virtual uint64_t size() const = 0;
virtual void resize(uint64_t numBytes) = 0;
virtual cpputils::Data readAll() const = 0;
virtual void read(void *target, uint64_t offset, uint64_t size) const = 0;
virtual uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const = 0;
virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
virtual void flush() = 0;
//TODO Test tryRead
};
}
#endif
src/blobstore/interface/BlobStore.h 0000664 0000000 0000000 00000001702 13235746211 0017650 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
#define MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
#include "Blob.h"
#include
#include
#include
#include
namespace blobstore {
//TODO Remove this interface. We'll only use BlobStoreOnBlocks and never a different one. Rename BlobStoreOnBlocks to simply BlobStore.
class BlobStore {
public:
virtual ~BlobStore() {}
virtual cpputils::unique_ref create() = 0;
virtual boost::optional> load(const blockstore::Key &key) = 0;
virtual void remove(cpputils::unique_ref blob) = 0;
virtual uint64_t numBlocks() const = 0;
virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0;
//virtual means "space we can use" as opposed to "space it takes on the disk" (i.e. virtual is without headers, checksums, ...)
virtual uint64_t virtualBlocksizeBytes() const = 0;
};
}
#endif
src/blockstore/ 0000775 0000000 0000000 00000000000 13235746211 0014020 5 ustar 00root root 0000000 0000000 src/blockstore/CMakeLists.txt 0000664 0000000 0000000 00000002726 13235746211 0016567 0 ustar 00root root 0000000 0000000 project (blockstore)
set(SOURCES
utils/Key.cpp
utils/BlockStoreUtils.cpp
utils/FileDoesntExistException.cpp
interface/helpers/BlockStoreWithRandomKeys.cpp
implementations/testfake/FakeBlockStore.cpp
implementations/testfake/FakeBlock.cpp
implementations/inmemory/InMemoryBlock.cpp
implementations/inmemory/InMemoryBlockStore.cpp
implementations/parallelaccess/ParallelAccessBlockStore.cpp
implementations/parallelaccess/BlockRef.cpp
implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
implementations/compressing/CompressingBlockStore.cpp
implementations/compressing/CompressedBlock.cpp
implementations/compressing/compressors/RunLengthEncoding.cpp
implementations/compressing/compressors/Gzip.cpp
implementations/encrypted/EncryptedBlockStore.cpp
implementations/encrypted/EncryptedBlock.cpp
implementations/ondisk/OnDiskBlockStore.cpp
implementations/ondisk/OnDiskBlock.cpp
implementations/caching/CachingBlockStore.cpp
implementations/caching/cache/PeriodicTask.cpp
implementations/caching/cache/CacheEntry.cpp
implementations/caching/cache/Cache.cpp
implementations/caching/cache/QueueMap.cpp
implementations/caching/CachedBlock.cpp
implementations/caching/NewBlock.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})
src/blockstore/implementations/ 0000775 0000000 0000000 00000000000 13235746211 0017230 5 ustar 00root root 0000000 0000000 src/blockstore/implementations/caching/ 0000775 0000000 0000000 00000000000 13235746211 0020624 5 ustar 00root root 0000000 0000000 src/blockstore/implementations/caching/CachedBlock.cpp 0000664 0000000 0000000 00000001661 13235746211 0023456 0 ustar 00root root 0000000 0000000 #include "CachedBlock.h"
#include "CachingBlockStore.h"
using cpputils::unique_ref;
namespace blockstore {
namespace caching {
CachedBlock::CachedBlock(unique_ref baseBlock, CachingBlockStore *blockStore)
:Block(baseBlock->key()),
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)) {
}
CachedBlock::~CachedBlock() {
if (_baseBlock.get() != nullptr) {
_blockStore->release(std::move(_baseBlock));
}
}
const void *CachedBlock::data() const {
return _baseBlock->data();
}
void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
return _baseBlock->write(source, offset, size);
}
void CachedBlock::flush() {
return _baseBlock->flush();
}
size_t CachedBlock::size() const {
return _baseBlock->size();
}
void CachedBlock::resize(size_t newSize) {
return _baseBlock->resize(newSize);
}
unique_ref CachedBlock::releaseBlock() {
return std::move(_baseBlock);
}
}
}
src/blockstore/implementations/caching/CachedBlock.h 0000664 0000000 0000000 00000001655 13235746211 0023126 0 ustar 00root root 0000000 0000000 #pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#include "../../interface/Block.h"
#include
namespace blockstore {
namespace caching {
class CachingBlockStore;
class CachedBlock final: public Block {
public:
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
CachedBlock(cpputils::unique_ref baseBlock, CachingBlockStore *blockStore);
~CachedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref releaseBlock();
private:
CachingBlockStore *_blockStore;
cpputils::unique_ref