pax_global_header00006660000000000000000000000064145361702520014517gustar00rootroot0000000000000052 comment=9d7ddd0706fd291650f48e71becf7f017bf73129 genomicsdb-0.0~git20231212.9d7ddd0/000077500000000000000000000000001453617025200163455ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/.codecov.yml000066400000000000000000000005431453617025200205720ustar00rootroot00000000000000ignore: - "examples/" - "test/" parsers: gcov: branch_detection: conditional: no loop: yes method: no macro: no coverage: status: project: default: target: auto threshold: 5% branches: null patch: default: target: auto branches: null changes: false genomicsdb-0.0~git20231212.9d7ddd0/.github/000077500000000000000000000000001453617025200177055ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/000077500000000000000000000000001453617025200213745ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/aws.tar.gpg000066400000000000000000000005761453617025200234620ustar00rootroot00000000000000   c3qj+ؚ M{fx dݤb9b67GN*I &%fvٱIR"r9UUƀxPWZɵZY|99Ph !UEGy^GH 4U2o TNU8S?/%KŎ;5Xt4J5.(qXXi句pM`;Hk QWگ00dB߷+9~oLso>ܭ C͈@Q RA{ e24U![= >E&{JLagenomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/azurite.tar.gpg000066400000000000000000000065731453617025200243560ustar00rootroot00000000000000  (4I߈2Чz=u{b -uiԼz}Z5d|P[y8@Ç5D3bQGY@}ѼG&xh('R,P@wlk^.MkW Dzu'Az8'.¾;._KCQFNh(nUU_K?Jo ~dN~<zv&M0i&l0twt1B1"ꇂhzzok֜ÝJbT\AT\* h;JgYf}wW<^gV-,w ݢ%KQ~0VOX-Ng7)-Tg}4etb_*Ca !yCH  c\#1*n<`;j1Uَg#a{ ++NU'1*R DOq<,Aa݄CIz@/#7ɦ-dϏk|q:U%0wuT~)hJ27A\Vb%dٕQ㐉:)aX\O`;dr#D$PmY=h{5*Y<5'PJ1 $٠ [ܞgb/BE;*.jc-)K Μ}`z j?RkG"Q;X7TQ8n ?rASjmI4 X/T8 fФ+j_K\yB<ݱjS;(@hbKPT.-ņDf u\CRLo0qQFbYif 1zcě <6?YC j4z-%L mfr>ZFܑ 8TlD6vsܬ2|E>-x#pr1B$m5|O(9n{!5vDž B\L!cW;hmre;-9n;u:=YnL; i0.}hyx,c (>q<2̊qxl:ѓQEᄝB;Ǜ~窅uUthLBIh>%=o<~8:<->ZBq^^}k1EfH,S?,}bg}0[+- ~2(^+ۡc&"oٛ<*SPn҉iۮj u@d R0 %9qֲ]0znW,JSaO|MS|:!9_%ų}ޤZ"e9CVٹ_gS0x2+L.tb#Lx~pIX4X:f@$ y*o c6w!O)j5mLɶ#qI*{88 8ߊoiTBsh[1bر*□X+V}%`=*f44gA"I tYQs2ѥ,hRM6B5e5سSh rt3UaF@OtKIPl6)QBRš0wb uZ4:Nb.a]hDsN$ʤBxSIǍKȔɝ'lΒ6q _m*( VW#fszAbezdy#͆,ōY=nпV? ൜5ZQS6wɬŭ$,q9Ő@ܰo~=G h!MR/\.]mc/$#Z&a\ƞBrV7R@{C'Qh3͖ۭ,_K4|:j^/LYJj^Կ蟡oʥ1^;M`Cl*gDV[tfXK8Ӝ5`'D63C\ " }ly"JYpm*rMMi5r=\{J/QRGQș#/k~ՅGR#[Wf!OiAlIMn.4?;W0V +_JUlSDGњq/}щPl7mxu>\K$5,@ oy\wJN+^5v:Q -&%pAe@Y x| .eS~+t}ךR2w8G8h˳w_-0]V. duVƜgGɀb&׵198 <ڡfimHq82:JIам0xj_ʬrH3*%C%ra| }=a9=Q oVlj!-R.~Ԑr{^%j}㛫<ܹ_϶ȚȲW b EQJ>de$QihyP0bUOn!$ߋncPTb˃Nm#ૻĜpoe!4X8 bZhO9GЇ.genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_aws.sh000077500000000000000000000006201453617025200242510ustar00rootroot00000000000000#!/bin/bash # Error out on first failure set -e gpg --quiet --batch --yes --decrypt --passphrase=$AWS_TAR --output ${HOME}/aws.tar $GITHUB_WORKSPACE/.github/scripts/aws.tar.gpg tar xvf ${HOME}/aws.tar -C $HOME # Install aws cli for sanity checking if [[ ! -n $(which aws) ]]; then sudo apt-get -q update sudo apt-get -y install awscli fi # Sanity check echo "aws version=`aws --version`" genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_azurite.sh000077500000000000000000000012041453617025200251410ustar00rootroot00000000000000#!/bin/bash # Error out on first failure set -e sudo apt-get -y install nodejs echo "node version = `node --version`" sudo npm install -g azurite AZURITE_DIR=$INSTALL_DIR/azurite mkdir $AZURITE_DIR gpg --quiet --batch --yes --decrypt --passphrase=$AZURITE_TAR --output $AZURITE_DIR/azurite.tar $GITHUB_WORKSPACE/.github/scripts/azurite.tar.gpg ls $AZURITE_DIR tar xvf $AZURITE_DIR/azurite.tar -C $AZURITE_DIR sudo cp $AZURITE_DIR/cert.pem /usr/local/share/ca-certificates/ca-certificates.crt sudo update-ca-certificates which azurite # Start azurite azurite --location $AZURITE_DIR --cert $AZURITE_DIR/cert.pem --key $AZURITE_DIR/key1.pem & genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_catch2.sh000077500000000000000000000031731453617025200246310ustar00rootroot00000000000000# # install_catch2.sh # # The MIT License # # Copyright (c) 2022 Omics Data Automation, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # #!/bin/bash # Error out on first failure set -e INSTALL_DIR=${INSTALL_DIR:-/usr/local} CATCH2_VER=${CATCH2_VER:-v3.1.0} if [[ $INSTALL_DIR == "/usr/local" ]]; then SUDO="sudo" INSTALL_PREFIX="" else SUDO="" INSTALL_PREFIX="-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR" fi if [[ ! -d $INSTALL_DIR/include/catch2 ]]; then git clone https://github.com/catchorg/Catch2.git -b $CATCH2_VER cd Catch2 cmake -Bbuild -H. -DBUILD_TESTING=OFF $INSTALL_PREFIX $SUDO cmake --build build/ --target install fi genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_hadoop.sh000077500000000000000000000102161453617025200247330ustar00rootroot00000000000000#!/bin/bash # Install hadoop # Installation relies on finding JAVA_HOME@/usr/java/latest as a prerequisite INSTALL_DIR=${INSTALL_DIR:-/usr} USER=`whoami` HADOOP=hadoop-${HADOOP_VER:-3.2.2} HADOOP_DIR=${INSTALL_DIR}/$HADOOP HADOOP_ENV=$HADOOP_DIR/hadoop.env install_prereqs() { if [[ -f /usr/java/latest ]]; then echo "/usr/java/latest found" sudo rm /usr/java/latest fi if [[ ! -z $JAVA_HOME ]]; then sudo mkdir -p /usr/java sudo ln -s $JAVA_HOME /usr/java/latest else sudo apt install openjdk-8-jre-headless sudo ln -s /usr/lib/jvm/java-1.8.0-openjdk-amd64/ /usr/java/latest fi echo "install_prereqs successful" } # retry logic from: https://docs.microsoft.com/en-us/azure/hdinsight/hdinsight-hadoop-script-actions-linux MAXATTEMPTS=3 retry() { local -r CMD="$@" local -i ATTEMPTNUM=1 local -i RETRYINTERVAL=2 until $CMD do if (( ATTEMPTNUM == MAXATTEMPTS )) then echo "Attempt $ATTEMPTNUM failed. no more attempts left." return 1 else echo "Attempt $ATTEMPTNUM failed! Retrying in $RETRYINTERVAL seconds..." sleep $(( RETRYINTERVAL )) ATTEMPTNUM=$ATTEMPTNUM+1 fi done } download_gcs_connector() { if [[ $INSTALL_TYPE == gcs ]]; then retry wget -nv --trust-server-names https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop3-latest.jar mv gcs-connector-hadoop3-latest.jar ${HADOOP_DIR}/share/hadoop/common echo "download_gcs_connector successful" fi } download_hadoop() { retry wget -nv --trust-server-names https://archive.apache.org/dist/hadoop/common/$HADOOP/$HADOOP.tar.gz tar -xzf $HADOOP.tar.gz --directory $INSTALL_DIR && download_gcs_connector && echo "download_hadoop successful" } configure_passphraseless_ssh() { sudo apt update; sudo apt -y install openssh-server cat > sshd_config << EOF SyslogFacility AUTHPRIV PermitRootLogin yes AuthorizedKeysFile .ssh/authorized_keys PasswordAuthentication yes ChallengeResponseAuthentication no UsePAM yes UseDNS no X11Forwarding no PrintMotd no EOF sudo mv sshd_config /etc/ssh/sshd_config && sudo systemctl restart ssh && ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa && cat ~/.ssh/id_rsa.pub | tee -a ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys && chmod 700 ~/.ssh && sudo chmod -c 0755 ~/ && echo "configure_passphraseless_ssh successful" } configure_hadoop() { configure_passphraseless_ssh && $HADOOP_DIR/bin/hdfs namenode -format && $HADOOP_DIR/sbin/start-dfs.sh && if [[ $INSTALL_TYPE == gcs ]]; then export GOOGLE_APPLICATION_CREDENTIALS=$GITHUB_WORKSPACE/.github/resources/gcs/GCS.json source $GITHUB_WORKSPACE/.github/resources/gcs/gcs_cred.sh fi if [[ $INSTALL_TYPE == azure ]]; then source $GITHUB_WORKSPACE/.github/resources/azure/azure_cred.sh fi echo "configure_hadoop successful" } setup_paths() { echo "export JAVA_HOME=/usr/java/latest" > $HADOOP_ENV echo "export PATH=$HADOOP_DIR/bin:$PATH" >> $HADOOP_ENV echo "export LD_LIBRARY_PATH=$HADOOP_DIR/lib:$LD_LIBRARY_PATH" >> $HADOOP_ENV HADOOP_CP=`$HADOOP_DIR/bin/hadoop classpath --glob` AZURE_JARS=`find $HADOOP_DIR/share/hadoop/tools/lib -name *azure*jar | tr '\n' ':'` echo "export CLASSPATH=$AZURE_JARS$HADOOP_CP" >> $HADOOP_ENV echo "setup_paths successful" } install_hadoop() { echo "Installing Hadoop..." install_prereqs if [[ ! -f $HADOOP_ENV ]]; then download_hadoop && setup_paths && cp -fr $GITHUB_WORKSPACE/.github/resources/hadoop/* $HADOOP_DIR/etc/hadoop && mkdir -p $HADOOP_DIR/logs && export HADOOP_ROOT_LOGGER=ERROR,console fi source $HADOOP_ENV && configure_hadoop && echo "Install Hadoop SUCCESSFUL" } echo "INSTALL_DIR=$INSTALL_DIR" echo "INSTALL_TYPE=$INSTALL_TYPE" # resources r.tar file encrypted using "gpg --symmetric --cipher-algo AES256 r.tar" gpg --quiet --batch --yes --decrypt --passphrase="$R_TAR" --output $INSTALL_DIR/r.tar $GITHUB_WORKSPACE/.github/scripts/r.tar.gpg && tar xf $INSTALL_DIR/r.tar -C $GITHUB_WORKSPACE/.github && install_hadoop genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_minio.sh000077500000000000000000000035161453617025200246010ustar00rootroot00000000000000#!/bin/bash # Error out on first failure set -e setup_env() { export AWS_ACCESS_KEY_ID=minio_s3 > $HOME/aws_env.sh export AWS_SECRET_ACCESS_KEY=minio_s3_secret_key >> $HOME/aws_env.sh export AWS_DEFAULT_REGION=us-west-2 >> $HOME/aws_env.sh export MINIO_ACCESS_KEY=$AWS_ACCESS_KEY_ID >> $HOME/aws_env.sh export MINIO_SECRET_KEY=$AWS_SECRET_ACCESS_KEY >> $HOME/aws_env.sh export AWS_CA_BUNDLE=${HOME}/.minio/certs/cert.pem >> $HOME/aws_env.sh export AWS_ENDPOINT_URL=https://127.0.0.1:9000 >> $HOME/aws_env.sh export AWS_ENDPOINT_OVERRIDE=https://127.0.0.1:9000 >> $HOME/aws_env.sh } sudo wget -nv -P /usr/local/bin https://dl.min.io/server/minio/release/linux-amd64/minio sudo chmod +x /usr/local/bin/minio setup_env source $HOME/aws_env.sh # same certificates as used for azurite mkdir -p ${HOME}/.minio/certs gpg --quiet --batch --yes --decrypt --passphrase=$AZURITE_TAR --output ${HOME}/.minio/azurite.tar $GITHUB_WORKSPACE/.github/scripts/azurite.tar.gpg tar xvf ${HOME}/.minio/azurite.tar -C ${HOME}/.minio/certs cp ${HOME}/.minio/certs/cert.pem ${HOME}/.minio/certs/public.crt cp ${HOME}/.minio/certs/key1.pem ${HOME}/.minio/certs/private.key sudo cp ${HOME}/.minio/certs/public.crt /usr/local/share/ca-certificates/ca-certificates.crt sudo update-ca-certificates # Start minio mkdir -p /tmp/minio/data minio server /tmp/minio/data & # Test with curl sleep 10 echo "Testing with curl..." curl -vsSL --tlsv1.2 -X GET $AWS_ENDPOINT_URL echo "Testing with curl DONE" if [[ ! -n $(which aws) ]]; then sudo apt-get -q update sudo apt-get -y install awscli fi echo "aws version=`aws --version`" echo "Listing Buckets..." aws configure set default.s3.signature_version s3v4 aws --endpoint-url $AWS_ENDPOINT_URL s3 ls aws --endpoint-url $AWS_ENDPOINT_URL s3 mb s3://test aws --endpoint-url $AWS_ENDPOINT_URL s3 ls echo "Listing Buckets DONE" genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_openssl1.1.sh000077500000000000000000000007521453617025200253700ustar00rootroot00000000000000#!/bin/bash INSTALL_PREFIX=/usr/local OPENSSL_VERSION=1.1.1o OPENSSL_PREFIX=$INSTALL_PREFIX echo "Installing OpenSSL" pushd /tmp wget -nv https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz && tar -xvzf openssl-$OPENSSL_VERSION.tar.gz && cd openssl-$OPENSSL_VERSION && CFLAGS=-fPIC ./config shared --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX make && make install && echo "Installing OpenSSL DONE" rm -fr /tmp/openssl* popd export OPENSSL_ROOT_DIR=$OPENSSL_PREFIX genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/install_supported_codec.sh000077500000000000000000000006621453617025200266470ustar00rootroot00000000000000#!/bin/bash INSTALL_DIR=${INSTALL_DIR:-/usr} # Install ZStd sudo apt-get install -y zstd && export ENABLE_ZSTD=1 # Install Blosc echo "Installing Blosc..." pushd `pwd` git clone https://github.com/Blosc/c-blosc.git && cd c-blosc && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR .. && cmake --build . && sudo cmake --build . --target install && export ENABLE_BLOSC=1 popd echo "Installing Blosc DONE" genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/notes000066400000000000000000000010331453617025200224440ustar00rootroot00000000000000For generating certs to use with azurite and minio - ~/openssl1.1.1m/bin/openssl req -newkey rsa:2048 -x509 -nodes -keyout key.pem -new -out cert.pem -sha256 -days 365 -addext "subjectAltName=IP:127.0.0.1" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" ~/openssl1.1.1m/bin/openssl rsa -in key.pem -out key1.pem tar cvf azurite.tar cert.pem key.pem key1.pem Storing encrypted secrets gpg --symmetric --cipher-algo AES256 azurite.tar From /settings/secrets/actions, create/update secret AZURITE_TAR using the password used with gpg genomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/r.tar.gpg000066400000000000000000000131571453617025200231300ustar00rootroot00000000000000  t/'h %9ߪأ9MY}z3:&G:aU͟Um޸D٧tsD[6%'"3Mہhk![f e;":sLH|B ^[e!58WF!Z$!qȶ-y~-\F]]ĒGS ,s1uȬ| A:^Os{i'IcRRm|4;B؅sVdaZ6;P)$myn}P@H h"{< Xw%[yAm$Q}ɺZs[Tڢ .+A @B/yE _E̾XL7@Q4<"(}M(QfL9Gۊm,PH4PL.,7L8sh~G{i@}/VAԐ!i xRRnm"0 ;a7!*cXV(l ӽR}@;ji=X5㖐^\/R-Q/#TK;HZ61 vk7, =_ޏl+!磾MAΛ|0"Jj'A d&e#6gCJ|WsyQ;bK2ޜɧR)Zْ~pF1l 5wBakg-X2Tuh&,S Fk{3%qtd "bGy/xJ *+ cfT`˖}pj}ULWRF3tY1!"00z.Ye0F0f#O_`AMudp"%˕K N?\K=ܶ~IT#-ko ׯ7d?<5~ Db8٦/,hn+sT$w*DɁAg5?J')t/S+@*$HA6xޥ)%`Y4Wgx c9FI < QL*GHDUzTcKr&JVOPs9Vy>b7"fb^))OWDR).'/ F5l%/ kF#&7J½J"3ǵQNt[k.:\Y_X@ZZƫm8G̣H+&MJ:5>f3~2T_>cۆ 6Z㾱Nvo[; # / Ho]b{Dфl+iۦ҃"،v`R )h4g.LvXBFͱjv'5%ߴgSA6qfZޡCyAª~76h%K=`[bw>WFSҨ 2|+FTT81֭Sjl-uk_Y$tD}5#E|y28V6Wbp1|j 'Lf< d#*BNOȦ(:vf GZs?n#ͼIE/C)cӴ(?e۸VPFEfJü5WŊSfD%=a[$?ktnR?N9kBZݼб?_Q=J"A2b+2Z!VNL? 9J}~@'ƚ!10r9%+%+kqj9օt('L! D8~*Fawt| H7Ksy}7eZ^Gȸ ȓ*v1 e\rɖf^3=7|/'݀vMgEDsRɨ]D%!uz#|TďE1*&y8}NFB<0B&=W^ =,^?ٔd.Tk3UJKBH1\DOᒥv1,55ƜuLe)f+!G/ǟ5G|ӔKl{~0?`g@ZF m Ջ0mםԽCRᚮ@#A[`SR k˦(Rj v4bK|t(bb]Kݒ26tf}eI ~-~qR )p2$P![-x~aMЍ.'U߿mGXl3 h1iGAHARMY*S] OJףkflٳQ6޸d]eD0Uh >>{1 -X2b oX"#͊3 ' >ocNm[ Iu~u*y[fë>l;Z;7P%@CA>IK^^{R#jtoKUCQ_t*a eӵW|ی+& B6ś(yF;n+?| u=M#=.AO!Fu.l;ta\xj[|~q2k :`.l`6yo.;8~ FW6 ^f܃]x`cոS;kG\# Ӟiqy;釟_߯K])Z6Cl "$oS"z(oʵ+Yn44YH?V58c>F4)#͹aqap:%|/U48ik$ *y;,6 =qY؅NJQ;tg[LO揾>I a1]Y+O7;Zso-Si煃;3XvL*nTMbLz:Ihv>WL7|WQnJq-XbDqJyѥ5 +__0x.30eDYlrbHk[ۨe-6}<85aVqLfG]!L뭧 'TC\B9 -0Oj ID5f]Ls/r-g'H;W榱8XR=dԿ|WXE(BS Ž/J4~oZEutokR R#i0# UtY43,yp !#lRnacpjXH1 3(U<9ō{:j)&"P*$AhBFQtAOBlilM}<1ħ4\ [=!asc8LS "e_611s r̕]߰x$9V3@F5`LFsX(uleu x\1goMf•c|v/"g!G+:hyhT='##hzB5ۮF#iX(^%/OWD%uft;~Dd hCK9.)i:[!*pz)jޏ}wB`,Rox]%]oآK5"j@zquK2cE]M,n=svEMzehƲ[kUo;_:zE͖SAh$c\e ?)F:oH3.m:pznW}6όLXyXr1q{7ƧddcWSe, 76&4KPFH%lƺcZO[;& 1:SБZo{Lcmhr,"5\{?룝kGx1&D)H$\*t.|սFY}\vE؂J?&dIt Uq;sCeEW *-?%΁řmoeVC4*Ԝ~%V]T:*M71X)“y0]Vi#h^ kT(:iݗ.Yz* ƚ1˟0btq q!v?D/:< Sj| :!F*x+}k z4>mDbAњ^f&@?\,JrIϿLT%-l*ktJOn+E2#O{B&${Lv-%RY`}f&UK8τFq 8];#hZxW5@^) EMwA/X3EBx/%9%1>.#D DYQ aA>VMJVC^B:|j%]52dND-rFfyrj- '׹ <,!s?,h__B%Y8B$IQ'QMZ݉ʓjw;-ɂ) _L.o}蠹rI):V[14D2Fi; ܄$/ m{ÌLv ^tzpOMYU@Y;~dG B/[$bgenomicsdb-0.0~git20231212.9d7ddd0/.github/scripts/run_dfs_tests.sh000077500000000000000000000144501453617025200246210ustar00rootroot00000000000000#!/bin/bash tiledb_utils_tests() { $CMAKE_BUILD_DIR/test/test_tiledb_utils [initialize_workspace] --test-dir $1 && $CMAKE_BUILD_DIR/test/test_tiledb_utils [create_workspace] --test-dir $1 && $CMAKE_BUILD_DIR/test/test_tiledb_utils [array_exists] --test-dir $1 && $CMAKE_BUILD_DIR/test/test_tiledb_utils [get_fragment_names] --test-dir $1 && $CMAKE_BUILD_DIR/test/test_tiledb_utils [file_utils_multi_threads] --test-dir $1 && $CMAKE_BUILD_DIR/test/test_tiledb_utils [file_ops] --test-dir $1 && $CMAKE_BUILD_DIR/test/test_tiledb_utils [move_across_filesystems] --test-dir $1 } setup_azurite() { # TODO: Ignoring "SSL SecurityWarning: Certificate has no subjectAltName", see https://docs.oracle.com/cd/E52668_01/E66514/html/ceph-issues-24424028.html for a fix export AZURE_CLI_DISABLE_CONNECTION_VERIFICATION=anycontent export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # Create container called test as TileDB expects the container to be already created az storage container create -n test --connection-string "DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=https://127.0.0.1:10001/devstoreaccount1;" # Env to run tests export AZURE_STORAGE_ACCOUNT=devstoreaccount1 export AZURE_STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" export AZURE_STORAGE_SERVICE_ENDPOINT="https://127.0.0.1:10000/devstoreaccount1" } check_results_from_examples() { echo "check_results_from_examples for $TEST.log" if [[ -f $TEST.log ]]; then if diff $TEST.log $GITHUB_WORKSPACE/examples/expected_results; then echo "check_results_from_examples for $TEST.log DONE" exit 0 else echo "$TEST.log from run_examples.sh is different from expected results" exit 1 fi else echo "$TEST.log from run_examples.sh does not seem to exist. Check the results of running run_examples.sh" exit 1 fi } run_azure_tests() { source $1 echo "Running TEST_DIR=$TEST" echo "az schema utils test" && tiledb_utils_tests "az://$AZURE_CONTAINER_NAME@$AZURE_STORAGE_ACCOUNT.$3/$TEST" && tiledb_utils_tests "azb://$AZURE_CONTAINER_NAME/$TEST?endpoint=$AZURE_STORAGE_ACCOUNT.$3" && echo "az schema storage test" && $CMAKE_BUILD_DIR/test/test_azure_blob_storage --test-dir "az://$AZURE_CONTAINER_NAME@$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$TEST" && $CMAKE_BUILD_DIR/test/test_azure_blob_storage --test-dir "azb://$AZURE_CONTAINER_NAME/$TEST?account=$AZURE_STORAGE_ACCOUNT" && echo "az schema storage buffer test" && $CMAKE_BUILD_DIR/test/test_storage_buffer --test-dir "az://$AZURE_CONTAINER_NAME@$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$TEST" && echo "az schema examples" && time $GITHUB_WORKSPACE/examples/run_examples.sh "az://$AZURE_CONTAINER_NAME@$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$TEST" && echo "az schema small size storage test" && (TILEDB_MAX_STREAM_SIZE=32 $CMAKE_BUILD_DIR/test/test_azure_blob_storage --test-dir "az://$AZURE_CONTAINER_NAME@$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$TEST" [read-write-small] || echo "az schema small size storage test failed") && echo "Running with $1 DONE" && check_results_from_examples } make -j4 && make test_tiledb_utils && make test_azure_blob_storage && make test_sparse_array_benchmark && make test_s3_storage && make test_gcs_storage && make test_storage_buffer make examples if [ $? -ne 0 ]; then echo "Errors encountered during TileDB build" exit 1 fi cd examples TEST=github_test_$RANDOM if [[ $INSTALL_TYPE == hdfs ]]; then echo "JAVA_HOME=$JAVA_HOME" java -version echo "1 CLASSPATH=$CLASSPATH" tiledb_utils_tests "hdfs://localhost:9000/$TEST" && time $GITHUB_WORKSPACE/examples/run_examples.sh "hdfs://localhost:9000/$TEST" elif [[ $INSTALL_TYPE == gcs ]]; then tiledb_utils_tests "gs://$GS_BUCKET/$TEST" && $CMAKE_BUILD_DIR/test/test_gcs_storage --test-dir "gs://$GS_BUCKET/$TEST" && $CMAKE_BUILD_DIR/test/test_storage_buffer --test-dir "gs://$GS_BUCKET/$TEST" && $GITHUB_WORKSPACE/examples/run_examples.sh "gs://$GS_BUCKET/$TEST" elif [[ $INSTALL_TYPE == azure ]]; then export AZURE_CONTAINER_NAME="build" CHECK_RESULTS=(-1 -1) run_azure_tests $GITHUB_WORKSPACE/.github/resources/azure/azure_cred.sh 0 "blob.core.windows.net" & pids[0]=$! TEST=github_test_${RANDOM}_adls run_azure_tests $GITHUB_WORKSPACE/.github/resources/azure/azure_cred_adls.sh 1 "dfs.core.windows.net" & pids[1]=$! wait ${pids[0]} CHECK_RESULTS[0]=$? wait ${pids[1]} CHECK_RESULTS[1]=$? echo "Finished running Azure tests" if [[ ${CHECK_RESULTS[0]} != 0 || ${CHECK_RESULTS[1]} != 0 ]]; then echo "Failure in some Azure tests: ${CHECK_RESULTS[0]} ${CHECK_RESULTS[1]}" exit 1 else exit 0 fi elif [[ $INSTALL_TYPE == azurite ]]; then setup_azurite tiledb_utils_tests "az://test@devstoreaccount1.blob/$TEST" && tiledb_utils_tests "azb://test/$TEST?account=devstoreaccount1&endpoint=https://127.0.0.1:10000/devstoreaccount1" && $CMAKE_BUILD_DIR/test/test_azure_blob_storage --test-dir "az://test@devstoreaccount1.blob/$TEST" && $CMAKE_BUILD_DIR/test/test_storage_buffer --test-dir "az://test@devstoreaccount1.blob/$TEST" && TEMP_VAR=$AZURE_STORAGE_ACCOUNT && unset AZURE_STORAGE_ACCOUNT && $CMAKE_BUILD_DIR/test/test_storage_buffer --test-dir "az://test@devstoreaccount1.blob/$TEST" && AZURE_STORAGE_ACCOUNT=$TEMP_VAR $GITHUB_WORKSPACE/examples/run_examples.sh "az://test@devstoreaccount1.blob/$TEST" && $GITHUB_WORKSPACE/examples/run_examples.sh "azb://test/$TEST?account=devstoreaccount1" elif [[ $INSTALL_TYPE == aws ]]; then TILEDB_BENCHMARK=1 tiledb_utils_tests "s3://github-actions-1/$TEST" && $CMAKE_BUILD_DIR/test/test_s3_storage --test-dir s3://github-actions-1/$TEST && $CMAKE_BUILD_DIR/test/test_storage_buffer --test-dir s3://github-actions-1/$TEST && $CMAKE_BUILD_DIR/test/test_sparse_array_benchmark --test-dir s3://github-actions-1/$TEST && $GITHUB_WORKSPACE/examples/run_examples.sh s3://github-actions-1/$TEST elif [[ $INSTALL_TYPE == minio ]]; then source $HOME/aws_env.sh && $CMAKE_BUILD_DIR/test/test_s3_storage --test-dir s3://test/$TEST && $GITHUB_WORKSPACE/examples/run_examples.sh s3://test/$TEST fi check_results_from_examples genomicsdb-0.0~git20231212.9d7ddd0/.github/workflows/000077500000000000000000000000001453617025200217425ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/.github/workflows/basic.yml000066400000000000000000000162741453617025200235600ustar00rootroot00000000000000# # basic.yml # # The MIT License # # Copyright (c) 2020-2023 Omics Data Automation, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # name: build on: push: paths-ignore: - '**/*.md' pull_request: paths-ignore: - '**/*.md' env: CMAKE_BUILD_TYPE: Coverage CMAKE_BUILD_DIR: ${{github.workspace}}/build HADOOP_VER: 3.2.4 AWSSDK_VER: 1.8.x GCSSDK_VER: v1.24.0 # Only used to install Catch2 on Ubuntu from source # MacOS uses brew that installs v3.1.0 # Ubuntu distributions are currently on v2.13.9 CATCH2_VER: v2.13.9 jobs: build: strategy: matrix: os: [ubuntu-20.04] # All supported types type: [basic, basic-no-hdfs, basic-codec, hdfs, gcs, azure, azurite, aws, minio] include: - os: macos-11 type: basic openssl-version: 1.1 - os: macos-11 type: basic openssl-version: 3 - os: macos-12 type: basic openssl-version: 1.1 - os: macos-12 type: basic openssl-version: 3 - os: ubuntu-22.04 type: basic - os: ubuntu-22.04 type: azurite - os: ubuntu-22.04 type: aws - os: ubuntu-22.04 type: gcs - os: ubuntu-22.04 type: azure runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - name: Create Build Environment shell: bash run: | mkdir -p $CMAKE_BUILD_DIR # For extracting cached Catch2 sudo chmod a+rwx /usr/local/include /usr/local/lib /usr/local/share /usr/local/share/doc mkdir -p /usr/local/share/vcpkg/ports - name: Cache AWS SDK uses: actions/cache@v3 with: path: ~/awssdk-install key: awssdk-${{env.AWSSDK_VER}}-${{matrix.os}}-openssl-${{matrix.openssl-version}}-v1 - name: Cache GCS SDK uses: actions/cache@v3 with: path: ~/gcssdk-install key: gcssdk-${{env.GCSSDK_VER}}-${{matrix.os}}-openssl-${{matrix.openssl-version}}-v1 - name: Cache Catch2 artifacts if: startsWith(matrix.os,'ubuntu') uses: actions/cache@v3 with: path: ~/catch2-install key: catch2-${{env.CATCH2_VER}}-v0-${{matrix.os}} - name: Cache Distributed FileSystems if: matrix.type == 'hdfs' || matrix.type == 'gcs' uses: actions/cache@v3 with: path: ${{runner.workspace}}/hadoop-${{env.HADOOP_VER}} key: dfs-${{env.HADOOP_VER}}-v1-${{matrix.os}}-openssl-v1 - name: Install Prereqs and Configure CMake - Ubuntu if: startsWith(matrix.os,'ubuntu') shell: bash working-directory: ${{env.CMAKE_BUILD_DIR}} run: | sudo apt-get update -q sudo apt-get -y install cmake lcov sudo apt-get -y install zlib1g-dev libssl-dev uuid-dev libcurl4-openssl-dev # sudo apt-get -y install catch2 INSTALL_DIR=~/catch2-install CATCH2_VER=$CATCH2_VER $GITHUB_WORKSPACE/.github/scripts/install_catch2.sh export ENABLE_ZSTD=0 export ENABLE_BLOSC=0 if [[ ${{matrix.type}} == "basic-no-hdfs" ]]; then export USE_HDFS="-DUSE_HDFS=0" fi if [[ ${{matrix.type}} == "basic-codec" ]]; then source $GITHUB_WORKSPACE/.github/scripts/install_supported_codec.sh fi echo "CMAKE VERSION=$(cmake --version)" cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DCMAKE_PREFIX_PATH=~/catch2-install \ -DENABLE_ZSTD=$ENABLE_ZSTD -DENABLE_BLOSC=$ENABLE_BLOSC \ -DAWSSDK_VERSION=$AWSSDK_VER -DGCSSDK_VERSION=$GCSSDK_VER $USE_HDFS - name: Install Prereqs and Configure CMake - MacOS if: startsWith(matrix.os, 'macOS') shell: bash working-directory: ${{env.CMAKE_BUILD_DIR}} run: | brew list cmake &>/dev/null || brew install cmake brew list lcov &>/dev/null || brew install lcov brew list openssl@${{matrix.openssl-version}} &>/dev/null || brew install openssl@${{matrix.openssl-version}} brew list ossp-uuid &>/dev/null || brew install ossp-uuid brew list catch2 &>/dev/null || brew install catch2 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DAWSSDK_VERSION=$AWSSDK_VER \ -DGCSSDK_VERSION=$GCSSDK_VER # GCS SDK needs OPENSSL_ROOT_DIR set for MAC, so build here make -j4 env: OPENSSL_ROOT_DIR: /usr/local/opt/openssl@${{matrix.openssl-version}} - name: Build and Test if: startsWith(matrix.type, 'basic') shell: bash working-directory: ${{env.CMAKE_BUILD_DIR}} run: | make -j4 make tests -j 4 export TILEDB_BENCHMARK=1 time test/test_sparse_array_benchmark export TILEDB_DISABLE_FILE_LOCKING=1 time test/test_sparse_array_benchmark export TILEDB_KEEP_FILE_HANDLES_OPEN=1 time test/test_sparse_array_benchmark test/test_print_array_schema $GITHUB_WORKSPACE/test/inputs/examples_ws/sparse_arrays/my_array_B test/test_print_book_keeping $GITHUB_WORKSPACE/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620 - name: Build and Test - Distributed FileSystems if: startsWith(matrix.type, 'basic') != true shell: bash working-directory: ${{env.CMAKE_BUILD_DIR}} run: | if [[ ${{matrix.type}} == "azurite" ]]; then source $GITHUB_WORKSPACE/.github/scripts/install_azurite.sh elif [[ ${{matrix.type}} == "minio" ]]; then source $GITHUB_WORKSPACE/.github/scripts/install_minio.sh elif [[ ${{matrix.type}} == "aws" ]]; then source $GITHUB_WORKSPACE/.github/scripts/install_aws.sh else source $GITHUB_WORKSPACE/.github/scripts/install_hadoop.sh fi source $GITHUB_WORKSPACE/.github/scripts/run_dfs_tests.sh env: INSTALL_DIR: ${{runner.workspace}} INSTALL_TYPE: ${{matrix.type}} R_TAR: ${{secrets.R_NEW_TAR}} AZURITE_TAR: ${{secrets.AZURITE_TAR}} AWS_TAR: ${{secrets.AWS_TAR}} - name: Upload Report to CodeCov uses: codecov/codecov-action@v3 with: gcov: true genomicsdb-0.0~git20231212.9d7ddd0/.gitignore000066400000000000000000000004151453617025200203350ustar00rootroot00000000000000*.swp core/bin/* core/obj/* core/lib/* core/include/temp core/src/temp coverage.info test/obj/* test/bin/* Doxyfile.log doxyfile.inc doxygen/html/ doxygen/latex/ examples/bin/* examples/obj/* *temp* *.DS_Store build #emacs *~ \#*\# .\#* TAGS #idea .idea/ TileDB.iml genomicsdb-0.0~git20231212.9d7ddd0/.gitmodules000066400000000000000000000005441453617025200205250ustar00rootroot00000000000000[submodule "deps/muparserx"] path = deps/muparserx url = https://github.com/beltoforion/muparserx.git [submodule "deps/HDFSWrapper"] path = deps/HDFSWrapper url = https://github.com/datma-health/HDFSWrapper.git [submodule "deps/azure-storage-cpplite"] path = deps/azure-storage-cpplite url = https://github.com/datma-health/azure-storage-cpplite.git genomicsdb-0.0~git20231212.9d7ddd0/CMakeLists.txt000066400000000000000000000261551453617025200211160ustar00rootroot00000000000000# # CMakeLists.txt # # # The MIT License # # Copyright (c) 2016 MIT and Intel Corporation # Copyright (c) 2018-2022 Omics Data Automation, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # cmake_minimum_required(VERSION 3.6) project(TileDB) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/Modules") # Update git submodules execute_process(COMMAND git submodule update --recursive --init WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} RESULT_VARIABLE submodule_update_exit_code) if(NOT(submodule_update_exit_code EQUAL 0)) message(FATAL_ERROR "Failure to recursively update git submodules") endif() set(ENABLE_BLOSC False CACHE BOOL "Enable blosc compression") set(ENABLE_ZSTD False CACHE BOOL "Enable zstd compression") set(LIBUUID_DIR "" CACHE PATH "Path to libuuid install directory") set(HDFS_SOURCE_DIR "deps/HDFSWrapper/hadoop-hdfs-native" CACHE PATH "Path to modified libhdfs source dir") set(AZURE_FSCLIENT_SOURCE_DIR "deps/azure-storage-cpplite" CACHE PATH "Path to modified azure cpp client") set(MUPARSERX_SOURCE_DIR "deps/muparserx" CACHE PATH "Path to muparserx source") set(AWSSDK_VERSION "1.8.x" CACHE STRING "Version to aws sdk") set(AWSSDK_ROOT_DIR "$ENV{HOME}/awssdk-install/${AWSSDK_VERSION}" CACHE PATH "Path to aws sdk") set(AWSSDK_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWSSDK_VERSION}.zip" CACHE STRING "URL to aws-sdk-cpp github release") set(GCSSDK_VERSION "v1.24.0" CACHE STRING "Version to gcs sdk") set(GCSSDK_ROOT_DIR "$ENV{HOME}/gcssdk-install/${GCSSDK_VERSION}" CACHE PATH "Path to gcs sdk") set(GCSSDK_URL "https://github.com/googleapis/google-cloud-cpp/archive/${GCSSDK_VERSION}.zip" CACHE STRING "URL to google-cloud-cpp github release") # Set the following to true downstream when TileDB is added as a subdirectory set(TILEDB_DISABLE_TESTS False CACHE BOOL "Disable TileDB Testing") set(TILEDB_DISABLE_UNINSTALL False CACHE BOOL "Disable TileDB Uninstall") # Only for Mac OS X if(APPLE) set(CMAKE_MACOSX_RPATH True CACHE BOOL "Set rpath on OSX") set(CMAKE_FIND_FRAMEWORK LAST CACHE STRING "Find frameworks on Mac OS X last") endif() # Default user definitions set(USE_MPI False CACHE BOOL "Enables MPI") if(APPLE) set(USE_OPENMP False CACHE BOOL "Enables OpenMP") else() set(USE_OPENMP True CACHE BOOL "Enables OpenMP") endif() if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) set(TILEDB_VERBOSE False CACHE BOOL "Prints TileDB errors with verbosity") else() set(TILEDB_VERBOSE True CACHE BOOL "Prints TileDB errors with verbosity") endif() set(TILEDB_TRACE False CACHE BOOL "Trace TileDB C API") set(USE_PARALLEL_SORT False CACHE BOOL "Enables parallel sorting.") set(USE_HDFS True CACHE BOOL "Enables HDFS support") set(COMPRESSION_LEVEL_GZIP "" CACHE STRING "Compression level for GZIP.") set(COMPRESSION_LEVEL_ZSTD "" CACHE STRING "Compression level for Zstandard.") set(COMPRESSION_LEVEL_BLOSC "" CACHE STRING "Compression level for Blosc.") set(DO_MEMORY_PROFILING False CACHE BOOL "Collect memory consumption in parts of TileDB - high overhead") # The DOWNLOAD_EXTRACT_TIMESTAMP option to the ExternalProject_Add() command is used to explicitly specify # how timestamps should be handled. The default behavior is to set the timestamps of extracted contents to # the time of extraction. # https://cmake.org/cmake/help/latest/policy/CMP0135.html#policy:CMP0135 if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() # Find required library dependencies find_package(ZLIB REQUIRED) include_directories(${ZLIB_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${ZLIB_LIBRARIES}) find_package(libuuid REQUIRED) include_directories(${LIBUUID_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${LIBUUID_LIBRARY}) # LZ4 source is directly included in source see /codec/external # find_package(LZ4) # include_directories(${LZ4_INCLUDE_DIR}) # set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${LZ4_LIBRARIES}) # Blosc and ZStd are dynamically loaded, so no need to perform find_package if(ENABLE_BLOSC) # find_package(BLOSC REQUIRED) # include_directories(${BLOSC_INCLUDE_DIR}) # set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${BLOSC_LIBRARIES}) add_definitions(-DENABLE_BLOSC) endif() if(ENABLE_ZSTD) # find_package(ZSTD REQUIRED) # include_directories(${ZSTD_INCLUDE_DIR}) # set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${ZSTD_LIBRARIES}) add_definitions(-DENABLE_ZSTD) endif() find_package(OpenSSL REQUIRED) include_directories(${OPENSSL_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${OPENSSL_LIBRARIES}) find_package(CURL REQUIRED) include_directories(${CURL_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${CURL_LIBRARIES}) # Build Azure cpp lite client added as a dependency include_directories(${AZURE_FSCLIENT_SOURCE_DIR}/include ${AZURE_FSCLIENT_SOURCE_DIR}/adls/include) add_subdirectory(${AZURE_FSCLIENT_SOURCE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} azure-storage-adls) # Build AWS S3 client static libraries find_package(S3Client REQUIRED) include_directories(${AWSSDK_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${AWSSDK_LINK_LIBRARIES}) # Build GCS Storage client static libraries find_package(GCSClient REQUIRED) include_directories(${GCSSDK_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${GCSSDK_LINK_LIBRARIES}) # libaws-c-common.a needs dl libs set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${CMAKE_DL_LIBS}) # Build MuParserX source code added as a dependency include_directories(${MUPARSERX_SOURCE_DIR}/parser) add_subdirectory(${MUPARSERX_SOURCE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} muparserx) # Find optional library dependencies find_package(Doxygen) if(USE_MPI) find_package(MPI REQUIRED) include_directories(${MPI_CXX_INCLUDE_PATH}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${MPI_CXX_LIBRARIES}) endif() if(USE_OPENMP AND NOT APPLE) find_package(OpenMP REQUIRED) endif() # Add HDFS support if(USE_HDFS) find_package(HDFS REQUIRED) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${JAVA_JVM_LIBRARY}) endif() # Add pthreads as a dependency set(CMAKE_THREAD_PREFER_PTHREAD True) find_package(Threads REQUIRED) if(DEFINED CMAKE_THREAD_LIBS_INIT AND (CMAKE_USE_PTHREADS_INIT OR CMAKE_HP_PTHREADS_INIT)) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${CMAKE_THREAD_LIBS_INIT}) endif() # TODO: Fix this - AWSSDK has some sort of transitive dependency on CoreFoundation, # add this explicitly for now even though CMAKE_FIND_FRAMEWORK is set to LAST if(APPLE) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} "-framework CoreFoundation") endif() # Set C++17 as required standard for all C++ targets. set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) # Set compiler flags set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -g3 -gdwarf-3 -Wall") set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 -fvisibility=hidden") set(CMAKE_CXX_FLAGS_COVERAGE "-DDEBUG -g3 -gdwarf-3 --coverage") if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-inconsistent-missing-override") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix") endif() if(USE_OPENMP AND NOT APPLE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() if (NOT APPLE) set(CMAKE_C_FLAGS "-std=gnu11") endif() # Add definitions add_definitions(-D_FILE_OFFSET_BITS=64) if(USE_MPI) add_definitions(-DHAVE_MPI) endif() if(USE_OPENMP AND NOT APPLE) add_definitions(-DHAVE_OPENMP) endif() if(TILEDB_VERBOSE) add_definitions(-DTILEDB_VERBOSE) message(STATUS "The TileDB library is compiled with verbosity.") endif() if(TILEDB_TRACE) if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_definitions(-DTILEDB_TRACE) message(STATUS "The TileDB library is compiled with trace.") else() message(STATUS "TileDB in release mode cannot have trace. CMAKE_BUILD_TYPE=" ${CMAKE_BUILD_TYPE}) endif() endif() if(USE_PARALLEL_SORT) add_definitions(-DUSE_PARALLEL_SORT) message(STATUS "Will use parallel sort.") endif() if(COMPRESSION_LEVEL_GZIP) add_definitions(-DTILEDB_COMPRESSION_LEVEL_GZIP=${COMPRESSION_LEVEL_GZIP}) message(STATUS "Set GZIP compression level to ${COMPRESSION_LEVEL_GZIP}.") endif() if(COMPRESSION_LEVEL_ZSTD) add_definitions(-DTILEDB_COMPRESSION_LEVEL_ZSTD=${COMPRESSION_LEVEL_ZSTD}) message(STATUS "Set Zstandard compression level to ${COMPRESSION_LEVEL_ZSTD}." ) endif() if(COMPRESSION_LEVEL_BLOSC) add_definitions(-DTILEDB_COMPRESSION_LEVEL_BLOSC=${COMPRESSION_LEVEL_BLOSC}) message(STATUS "Set Blosc compression level to ${COMPRESSION_LEVEL_BLOSC}.") endif() if(DO_MEMORY_PROFILING) message(STATUS "Enabling memory consumption profiling during consolidation") add_definitions(-DDO_MEMORY_PROFILING=1) endif() # Build TileDB library add_subdirectory(core) # Enable testing if(NOT TILEDB_DISABLE_TESTING) enable_testing() add_custom_target(tests COMMAND ${CMAKE_CTEST_COMMAND} -V) # Build unit tests add_subdirectory(test) endif() # Build examples add_subdirectory(examples) # Doxygen documentation if(DOXYGEN_FOUND) file(GLOB_RECURSE TILEDB_CORE_HEADERS "core/include/*.h") add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/doxyfile.in COMMAND mkdir -p doxygen COMMAND echo INPUT = ${CMAKE_SOURCE_DIR}/doc/mainpage.dox ${TILEDB_CORE_HEADERS} > ${CMAKE_BINARY_DIR}/doxyfile.in COMMAND echo FILE_PATTERNS = *.h >> ${CMAKE_BINARY_DIR}/doxyfile.in COMMENT "Preparing for Doxygen documentation" VERBATIM ) add_custom_target( doc ${DOXYGEN_EXECUTABLE} ${CMAKE_SOURCE_DIR}/doc/Doxyfile.mk > ${CMAKE_BINARY_DIR}/Doxyfile.log 2>&1 COMMENT "Generating API documentation with Doxygen" VERBATIM DEPENDS ${CMAKE_BINARY_DIR}/doxyfile.in ) endif(DOXYGEN_FOUND) # Uninstall if(NOT TILEDB_DISABLE_UNINSTALL) set(CMD "xargs printf '-- Uninstalling: %s\\\\n' x86_64") else() list(APPEND GCSSDK_COMMON_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}) endif() endif() # gcssdk has nlohmann json as its dependency ExternalProject_Add(nlohmann-build PREFIX ${GCSSDK_PREFIX} URL "https://github.com/nlohmann/json/archive/v3.9.1.zip" CMAKE_ARGS ${GCSSDK_COMMON_CMAKE_ARGS} -DJSON_BuildTests=OFF) # gcssdk has abseil as its dependency, so build that first ExternalProject_Add(abseil-build PREFIX ${GCSSDK_PREFIX} URL "https://github.com/abseil/abseil-cpp/archive/20230802.1.zip" # PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/gcssdk/absl.patch CMAKE_ARGS ${GCSSDK_COMMON_CMAKE_ARGS} -DABSL_PROPAGATE_CXX_STD=ON -DCMAKE_CXX_STANDARD=17) ExternalProject_Add(crc32-build PREFIX ${GCSSDK_PREFIX} URL "https://github.com/google/crc32c/archive/1.1.2.tar.gz" CMAKE_ARGS ${GCSSDK_COMMON_CMAKE_ARGS} -DCRC32C_BUILD_TESTS=OFF -DCRC32C_BUILD_BENCHMARKS=OFF -DCRC32C_USE_GLOG=OFF -DCMAKE_CXX_STANDARD=11) ExternalProject_Add(gcssdk-build PREFIX ${GCSSDK_PREFIX} URL ${GCSSDK_URL} PATCH_COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/core/include/misc/tiledb_openssl_shim.h ${GCSSDK_PREFIX}/src/gcssdk-build/google/cloud/storage && patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/gcssdk/gcs_ossl.patch BUILD_IN_SOURCE 1 CMAKE_ARGS ${GCSSDK_COMMON_CMAKE_ARGS} -DBUILD_TESTING=OFF -DGOOGLE_CLOUD_CPP_ENABLE=storage -DCMAKE_CXX_STANDARD=17 -DCMAKE_CXX_VISIBILITY_PRESET=hidden -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations") add_dependencies(gcssdk-build nlohmann-build) add_dependencies(gcssdk-build abseil-build) add_dependencies(gcssdk-build crc32-build) add_dependencies(gcssdk-ep gcssdk-build) endif() # Note: abseil is a mess with library organization... set(_GCSSDK_LIBS "google_cloud_cpp_storage" "google_cloud_cpp_common" "crc32c" "absl_bad_optional_access" "absl_bad_variant_access" "absl_strings_internal" "absl_str_format_internal" "absl_strings" "absl_throw_delegate" "absl_time" "absl_time_zone" "absl_raw_logging_internal" "absl_int128" ) set(GCSSSDK_LINK_LIBRARIES) file(MAKE_DIRECTORY ${GCSSDK_INCLUDE_DIR}) foreach(_GCSSDK_LIB ${_GCSSDK_LIBS}) set( _GCSSDK_STATIC_LIBRARY "${GCSSDK_PREFIX}/${GCSSDK_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_GCSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(_GCSSDK_TARGET_NAME GCS::${_GCSSDK_LIB}) add_library(${_GCSSDK_TARGET_NAME} STATIC IMPORTED) set_target_properties(${_GCSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION ${_GCSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES ${GCSSDK_INCLUDE_DIR}) list(APPEND GCSSDK_LINK_LIBRARIES ${_GCSSDK_TARGET_NAME}) endforeach() genomicsdb-0.0~git20231212.9d7ddd0/cmake/Modules/FindHDFS.cmake000066400000000000000000000033341453617025200233670ustar00rootroot00000000000000# # cmake/Modules/FindHDFS.cmake # # The MIT License # # Copyright (c) 2018 Omics Data Automation, Inc. and Intel Corporation. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # if(CMAKE_CROSSCOMPILING) set(JAVA_HOME ${TARGET_JAVA_HOME}) set(ENV{JAVA_HOME} ${TARGET_JAVA_HOME}) endif() find_package(JNI REQUIRED) if(CMAKE_CROSSCOMPILING) unset(JAVA_HOME) unset(ENV{JAVA_HOME}) endif() find_path(HDFS_INCLUDE_DIR hdfs.h HINTS ${HDFS_SOURCE_DIR}/main/native/libhdfs) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(HDFS "Could not find HDFS headers ${DEFAULT_MSG}" HDFS_INCLUDE_DIR) include_directories(${HDFS_INCLUDE_DIR}) add_definitions(-DUSE_HDFS) add_subdirectory(${HDFS_SOURCE_DIR} EXCLUDE_FROM_ALL) genomicsdb-0.0~git20231212.9d7ddd0/cmake/Modules/FindLZ4.cmake000066400000000000000000000034041453617025200232520ustar00rootroot00000000000000# # FindLZ4.cmake # # # The MIT License # # Copyright (c) 2016 MIT and Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Finds the LZ4 library. This module defines: # - LZ4_LIBRARIES, the LZ4 library path # - LZ4_FOUND, whether LZ4 has been found # Find library if(LZ4_SEARCH_LIB_PATH) find_library( LZ4_LIBRARIES NAMES lz4 PATHS ${LZ4_SEARCH_LIB_PATH}$ NO_DEFAULT_PATH ) else() find_library(LZ4_LIBRARIES NAMES lz4) endif() if(LZ4_LIBRARIES) message(STATUS "Found LZ4: ${LZ4_LIBRARIES}") set(LZ4_FOUND TRUE) else() message(STATUS "Did not find LZ4: build from lz4 sources in the repository") set(LZ4_FOUND FALSE) endif() if(LZ4_FIND_REQUIRED AND NOT LZ4_FOUND) message(FATAL_ERROR "Could not find the LZ4 library.") endif() genomicsdb-0.0~git20231212.9d7ddd0/cmake/Modules/FindS3Client.cmake000066400000000000000000000157171453617025200242770ustar00rootroot00000000000000# # FindS3Client.cmake # # The MIT License # # Copyright (c) 2022-2023 Omics Data Automation, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Finds and/or builds the s3 static library add_custom_target(awssdk-ep) message(CHECK_START "Finding AWS SDK Library") include(GNUInstallDirs) if(AWSSDK_ROOT_DIR) set(AWSSDK_PREFIX "${AWSSDK_ROOT_DIR}") elseif(DEFINED ENV{AWSSDK_ROOT_DIR}) set(AWSSDK_PREFIX $ENV{AWSSDK_ROOT_DIR}) endif() set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}") set(AWSSDK_LIB_DIR "${CMAKE_INSTALL_LIBDIR}") set(PWD $ENV{PWD}) set(S3_COMPONENTS config s3 transfer identity-management sts) if(EXISTS ${AWSSDK_PREFIX}/include AND EXISTS ${AWSSDK_PREFIX}/${CMAKE_INSTALL_LIBDIR}) get_filename_component(AWSSDK_PREFIX ${AWSSDK_PREFIX} ABSOLUTE BASE_DIR ${PWD}) set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${AWSSDK_PREFIX}) find_package(AWSSDK QUIET COMPONENTS ${S3_COMPONENTS} PATHS ${CMAKE_PREFIX_PATH}) else() find_package(AWSSDK QUIET COMPONENTS ${S3_COMPONENTS}) endif() if(AWSSDK_FOUND) message(STATUS "Found AWS SDK headers: ${AWSSDK_INCLUDE_DIR}") message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") message(CHECK_PASS "found") elseif(NOT AWSSDK_PREFIX) message(CHECK_FAIL "not found") message(FATAL_ERROR "Try invoking cmake with -DAWSSDK_ROOT_DIR=/path/to/awssdk or -DCMAKE_INSTALL_PREFIX=/path/to/awssdk or set environment variable AWSSDK_ROOT_DIR before invoking cmake") elseif(NOT AWSSDK_FOUND) # Try building from source message(STATUS "Adding AWS SDK as an external project") include(ExternalProject) set(AWSSDK_COMMON_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX} -DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX} -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}) # Workaround for issues with semicolon separated substrings to ExternalProject_Add # https://discourse.cmake.org/t/how-to-pass-cmake-osx-architectures-to-externalproject-add/2262 if(CMAKE_OSX_ARCHITECTURES) if(CMAKE_OSX_ARCHITECTURES MATCHES "x86_64" AND CMAKE_OSX_ARCHITECTURES MATCHES "arm64") list(APPEND AWSSDK_COMMON_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=arm64$x86_64) else() list(APPEND AWSSDK_COMMON_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}) endif() endif() ExternalProject_Add(aws-c-common-build URL "https://github.com/awslabs/aws-c-common/archive/v0.6.9.tar.gz" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) ExternalProject_Add(aws-checksums-build URL "https://github.com/awslabs/aws-checksums/archive/v0.1.12.tar.gz" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} DEPENDS aws-c-common-build) ExternalProject_Add(aws-c-event-stream-build URL "https://github.com/awslabs/aws-c-event-stream/archive/v0.1.5.tar.gz" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} DEPENDS aws-checksums-build) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized -Wno-deprecated-declarations") endif() ExternalProject_Add(awssdk-build PREFIX ${AWSSDK_PREFIX} URL ${AWSSDK_URL} PATCH_COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/core/include/misc/tiledb_openssl_shim.h ${AWSSDK_PREFIX}/src/awssdk-build/aws-cpp-sdk-core/include/aws/core/utils/crypto/openssl && patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/awssdk/build.patch && patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/awssdk/cjson.patch && patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/awssdk/eventstreamdecoder.patch && patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/awssdk/aws_ossl.patch CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} -DENABLE_TESTING=OFF -DENABLE_UNITY_BUILD=ON -DCUSTOM_MEMORY_MANAGEMENT=OFF -DBUILD_DEPS=OFF -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts -DMINIMIZE_SIZE=ON -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}) add_dependencies(awssdk-build aws-c-common-build) add_dependencies(awssdk-build aws-c-event-stream-build) add_dependencies(awssdk-build aws-checksums-build) add_dependencies(awssdk-ep awssdk-build) # AWS C++ SDK related libraries to link statically set(_AWSSDK_LIBS aws-cpp-sdk-identity-management aws-cpp-sdk-sts aws-cpp-sdk-cognito-identity aws-cpp-sdk-s3 aws-cpp-sdk-core aws-c-event-stream aws-checksums aws-c-common) set(AWSSDK_LINK_LIBRARIES) file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) # aws-c-common -> AWS-C-COMMON string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER) # AWS-C-COMMON -> AWS_C_COMMON string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER}) set( _AWSSDK_STATIC_LIBRARY "${AWSSDK_PREFIX}/${AWSSDK_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) else() set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB}) endif() add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED) set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION ${_AWSSDK_STATIC_LIBRARY} INTERFACE_INCLUDE_DIRECTORIES ${AWSSDK_INCLUDE_DIR}) list(APPEND AWSSDK_LINK_LIBRARIES ${_AWSSDK_TARGET_NAME}) endforeach() if(APPLE) # AWSSDK has some sort of transitive dependency on CoreFoundation, # add this explicitly even though CMAKE_FIND_FRAMEWORK is set to LAST list(APPEND AWSSDK_LINK_LIBRARIES "-framework CoreFoundation") endif() # aws-c libraries have a dl dependency list(APPEND AWSSDK_LINK_LIBRARIES dl) add_dependencies(aws-cpp-sdk-s3 awssdk-ep) message(STATUS "To be built AWS SDK headers: ${AWSSDK_INCLUDE_DIR}") message(STATUS "To be built AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") message(CHECK_PASS "AWS SDK to be built when make is invoked") endif() genomicsdb-0.0~git20231212.9d7ddd0/cmake/Modules/FindZSTD.cmake000066400000000000000000000040061453617025200234240ustar00rootroot00000000000000# # FindZSTD.cmake # # # The MIT License # # Copyright (c) 2016 MIT and Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Finds the Zstandard library. This module defines: # - ZSTD_INCLUDE_DIR, directory containing headers # - ZSTD_LIBRARIES, the Zstandard library path # - ZSTD_FOUND, whether Zstandard has been found # Find header files if(ZSTD_SEARCH_HEADER_PATHS) find_path( ZSTD_INCLUDE_DIR zstd.h PATHS ${ZSTD_SEARCH_HEADER_PATHS} NO_DEFAULT_PATH ) else() find_path(ZSTD_INCLUDE_DIR zstd.h) endif() # Find library if(ZSTD_SEARCH_LIB_PATH) find_library( ZSTD_LIBRARIES NAMES zstd PATHS ${ZSTD_SEARCH_LIB_PATH}$ NO_DEFAULT_PATH ) else() find_library(ZSTD_LIBRARIES NAMES zstd) endif() if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARIES) message(STATUS "Found Zstandard: ${ZSTD_LIBRARIES}") set(ZSTD_FOUND TRUE) else() set(ZSTD_FOUND FALSE) endif() if(ZSTD_FIND_REQUIRED AND NOT ZSTD_FOUND) message(FATAL_ERROR "Could not find the Zstandard library.") endif() genomicsdb-0.0~git20231212.9d7ddd0/cmake/Modules/Findlibuuid.cmake000066400000000000000000000035741453617025200243060ustar00rootroot00000000000000 # cmake Module for libuuid # # The MIT License # # Copyright (c) 2023 dātma, inc™ # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Determine compiler flags for libuuid # Once done this will define # LIBUUID_FOUND - libuuid found find_path(LIBUUID_INCLUDE_DIR NAMES uuid/uuid.h HINTS "${LIBUUID_DIR}/include" "${LIBUUID_DIR}") if(APPLE) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(libuuid "Could not find libuuid headers ${DEFAULT_MSG}" LIBUUID_INCLUDE_DIR) else() find_path(LIBUUID_INCLUDE_DIR NAMES uuid/uuid.h HINTS "${LIBUUID_DIR}/include" "${LIBUUID_DIR}") find_library(LIBUUID_LIBRARY NAMES uuid HINTS "${LIBUUID_DIR}/lib64" "${LIBUUID_DIR}/lib" "${LIBUUID_DIR}") include(FindPackageHandleStandardArgs) find_package_handle_standard_args(libuuid "Could not find libuuid headers and/or libraries ${DEFAULT_MSG}" LIBUUID_INCLUDE_DIR LIBUUID_LIBRARY) endif() genomicsdb-0.0~git20231212.9d7ddd0/cmake/Modules/SetCatch2Version.cmake000066400000000000000000000036751453617025200252000ustar00rootroot00000000000000# # cmake/Modules/SetCatch2Version.cmake # # The MIT License # # Copyright (c) 2022 Omics Data Automation, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Find Catch2 installation and setup Catch2 version to allow # test/include/Catch2/catch.h to support both major versions, 2 and 3 for now. # find_package(Catch2 REQUIRED) set(CATCH2_HEADER "catch2/catch.hpp") find_path(CATCH2_VER2_INCLUDE_FILE ${CATCH2_HEADER}) if(CATCH2_VER2_INCLUDE_FILE) set(CATCH2_MAJOR_VERSION 2) else() set(CATCH2_HEADER "catch2/catch_all.hpp") find_path(CATCH2_VER3_INCLUDE_FILE ${CATCH2_HEADER}) if(CATCH2_VER3_INCLUDE_FILE) set(CATCH2_MAJOR_VERSION 3) else() message(FATAL "Could not figure out Catch2 versions. Try using CMAKE_PREFIX_PATH to point to a Catch2 installation") endif() endif() message(STATUS "Found Catch2: ${CATCH2_INCLUDE_FILE}/${CATCH2_HEADER} Version=${CATCH2_MAJOR_VERSION}") add_definitions(-DCATCH2_MAJOR_VERSION=${CATCH2_MAJOR_VERSION}) genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/000077500000000000000000000000001453617025200210545ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/awssdk/000077500000000000000000000000001453617025200223505ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/awssdk/aws_ossl.patch000066400000000000000000000211721453617025200252260ustar00rootroot00000000000000--- awssdk-build-orig/aws-cpp-sdk-core/source/utils/crypto/openssl/CryptoImpl.cpp 2021-08-25 21:34:31.000000000 +0530 +++ awssdk-build/aws-cpp-sdk-core/source/utils/crypto/openssl/CryptoImpl.cpp 2023-03-18 11:49:08.011639537 +0530 @@ -5,10 +5,10 @@ #include +#include #include #include #include -#include #ifdef OPENSSL_IS_BORINGSSL #ifdef _MSC_VER @@ -18,13 +18,10 @@ #endif #endif -#include - #ifdef OPENSSL_IS_BORINGSSL AWS_SUPPRESS_WARNING_POP #endif -#include #include #include @@ -178,7 +175,13 @@ EVP_DigestInit_ex(ctx, EVP_md5(), nullptr); EVP_DigestUpdate(ctx, str.c_str(), str.size()); - ByteBuffer hash(EVP_MD_size(EVP_md5())); + size_t mdsize = 0; + if(OpenSSL_version_num() < 0x30000000L ) { + mdsize = EVP_MD_size(EVP_md5()); + } else { + mdsize = EVP_MD_get_size(EVP_md5()); + } + ByteBuffer hash(mdsize); EVP_DigestFinal(ctx, hash.GetUnderlyingData(), nullptr); return HashResult(std::move(hash)); @@ -216,7 +219,13 @@ stream.clear(); stream.seekg(currentPos, stream.beg); - ByteBuffer hash(EVP_MD_size(EVP_md5())); + size_t mdsize = 0; + if(OpenSSL_version_num() < 0x30000000L ) { + mdsize = EVP_MD_size(EVP_md5()); + } else { + mdsize = EVP_MD_get_size(EVP_md5()); + } + ByteBuffer hash(mdsize); EVP_DigestFinal(ctx, hash.GetUnderlyingData(), nullptr); return HashResult(std::move(hash)); @@ -228,8 +237,13 @@ auto ctx = guard.getResource(); EVP_DigestInit_ex(ctx, EVP_sha1(), nullptr); EVP_DigestUpdate(ctx, str.c_str(), str.size()); - - ByteBuffer hash(EVP_MD_size(EVP_sha1())); + size_t mdsize = 0; + if(OpenSSL_version_num() < 0x30000000L ) { + mdsize = EVP_MD_size(EVP_sha1()); + } else { + mdsize = EVP_MD_get_size(EVP_sha1()); + } + ByteBuffer hash(mdsize); EVP_DigestFinal(ctx, hash.GetUnderlyingData(), nullptr); return HashResult(std::move(hash)); @@ -266,7 +280,13 @@ stream.clear(); stream.seekg(currentPos, stream.beg); - ByteBuffer hash(EVP_MD_size(EVP_sha1())); + size_t mdsize = 0; + if(OpenSSL_version_num() < 0x30000000L ) { + mdsize = EVP_MD_size(EVP_sha1()); + } else { + mdsize = EVP_MD_get_size(EVP_sha1()); + } + ByteBuffer hash(mdsize); EVP_DigestFinal(ctx, hash.GetUnderlyingData(), nullptr); return HashResult(std::move(hash)); @@ -279,7 +299,13 @@ EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr); EVP_DigestUpdate(ctx, str.c_str(), str.size()); - ByteBuffer hash(EVP_MD_size(EVP_sha256())); + size_t mdsize = 0; + if(OpenSSL_version_num() < 0x30000000L ) { + mdsize = EVP_MD_size(EVP_sha256()); + } else { + mdsize = EVP_MD_get_size(EVP_sha256()); + } + ByteBuffer hash(mdsize); EVP_DigestFinal(ctx, hash.GetUnderlyingData(), nullptr); return HashResult(std::move(hash)); @@ -316,7 +342,13 @@ stream.clear(); stream.seekg(currentPos, stream.beg); - ByteBuffer hash(EVP_MD_size(EVP_sha256())); + size_t mdsize = 0; + if(OpenSSL_version_num() < 0x30000000L ) { + mdsize = EVP_MD_size(EVP_sha256()); + } else { + mdsize = EVP_MD_get_size(EVP_sha256()); + } + ByteBuffer hash(mdsize); EVP_DigestFinal(ctx, hash.GetUnderlyingData(), nullptr); return HashResult(std::move(hash)); @@ -328,25 +360,44 @@ #if OPENSSL_VERSION_LESS_1_1 m_ctx = Aws::New("AllocSha256HAMCOpenSSLContext"); #else - m_ctx = HMAC_CTX_new(); + if(OpenSSL_version_num() < 0x30000000L ) { + m_ctx = HMAC_CTX_new(); + assert(m_ctx != nullptr); + } else { + mac = EVP_MAC_fetch(NULL, "HMAC", NULL); + evp_ctx = EVP_MAC_CTX_new(mac); + assert(evp_ctx != nullptr); + } #endif - assert(m_ctx != nullptr); } ~HMACRAIIGuard() { #if OPENSSL_VERSION_LESS_1_1 Aws::Delete(m_ctx); #else - HMAC_CTX_free(m_ctx); + if (OpenSSL_version_num() < 0x30000000L ) { + HMAC_CTX_free(m_ctx); + m_ctx = nullptr; + } else { + EVP_MAC_CTX_free(evp_ctx); + evp_ctx = nullptr; + EVP_MAC_free(mac); + mac = nullptr; + } #endif - m_ctx = nullptr; } HMAC_CTX* getResource() { return m_ctx; } + + EVP_MAC_CTX* getEvpResource() { + return evp_ctx; + } private: - HMAC_CTX *m_ctx; + HMAC_CTX *m_ctx = nullptr; + EVP_MAC_CTX *evp_ctx = nullptr; + EVP_MAC *mac = nullptr; }; HashResult Sha256HMACOpenSSLImpl::Calculate(const ByteBuffer& toSign, const ByteBuffer& secret) @@ -356,21 +407,42 @@ memset(digest.GetUnderlyingData(), 0, length); HMACRAIIGuard guard; - HMAC_CTX* m_ctx = guard.getResource(); + HMAC_CTX *m_ctx = nullptr; + EVP_MAC_CTX *evp_ctx = nullptr; + if(OpenSSL_version_num() < 0x30000000L ) { + m_ctx = guard.getResource(); + } else { + evp_ctx = guard.getEvpResource(); + } #if OPENSSL_VERSION_LESS_1_1 HMAC_CTX_init(m_ctx); -#endif +#else - HMAC_Init_ex(m_ctx, secret.GetUnderlyingData(), static_cast(secret.GetLength()), EVP_sha256(), - NULL); + if(OpenSSL_version_num() < 0x30000000L ) { + HMAC_Init_ex(m_ctx, secret.GetUnderlyingData(), static_cast(secret.GetLength()), + EVP_sha256(), NULL); HMAC_Update(m_ctx, toSign.GetUnderlyingData(), toSign.GetLength()); HMAC_Final(m_ctx, digest.GetUnderlyingData(), &length); + } else { + char sha256[] {"SHA256"}; + OSSL_PARAM ossl_params[2]; + ossl_params[0] = + OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_DIGEST, sha256, 0); + ossl_params[1] = OSSL_PARAM_construct_end(); + EVP_MAC_init(evp_ctx, secret.GetUnderlyingData(), + static_cast(secret.GetLength()), ossl_params); + EVP_MAC_update(evp_ctx, toSign.GetUnderlyingData(), toSign.GetLength()); + EVP_MAC_final(evp_ctx, digest.GetUnderlyingData(), NULL, length); + } +#endif #if OPENSSL_VERSION_LESS_1_1 HMAC_CTX_cleanup(m_ctx); #else + if (OpenSSL_version_num() < 0x30000000L) { HMAC_CTX_reset(m_ctx); + } #endif return HashResult(std::move(digest)); } --- awssdk-build-orig/aws-cpp-sdk-core/include/aws/core/utils/crypto/openssl/CryptoImpl.h 2021-08-25 21:34:31.000000000 +0530 +++ awssdk-build/aws-cpp-sdk-core/include/aws/core/utils/crypto/openssl/CryptoImpl.h 2023-03-18 11:53:34.536554061 +0530 @@ -10,9 +10,6 @@ #include #include #include -#include -#include -#include #include #include genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/awssdk/build.patch000066400000000000000000000030221453617025200244650ustar00rootroot00000000000000diff -Naur awssdk-build/cmake/AWSSDKConfig.cmake awssdk-build.modify/cmake/AWSSDKConfig.cmake --- awssdk-build/cmake/AWSSDKConfig.cmake 2021-12-13 22:19:01.000000000 -0800 +++ awssdk-build.modify/cmake/AWSSDKConfig.cmake 2021-12-13 22:20:44.000000000 -0800 @@ -116,7 +116,7 @@ get_filename_component(TEMP_PATH "${AWSSDK_CORE_LIB_FILE}" PATH) get_filename_component(TEMP_NAME "${TEMP_PATH}" NAME) -while (NOT TEMP_NAME STREQUAL ${LIB_SEARCH_PREFIX}) +while (NOT TEMP_NAME STREQUAL "${LIB_SEARCH_PREFIX}") set(TEMP_PLATFORM_PREFIX "${TEMP_NAME}/${TEMP_PLATFORM_PREFIX}") get_filename_component(TEMP_PATH "${TEMP_PATH}" PATH) get_filename_component(TEMP_NAME "${TEMP_PATH}" NAME) diff -Naur awssdk-build/third-party/CMakeLists.txt awssdk-build.modify/third-party/CMakeLists.txt --- awssdk-build/third-party/CMakeLists.txt 2021-04-15 12:18:01.000000000 -0700 +++ awssdk-build.modify/third-party/CMakeLists.txt 2021-12-13 22:21:06.000000000 -0800 @@ -15,7 +15,7 @@ set(AWS_DEPS_DOWNLOAD_DIR "${AWS_DEPS_BUILD_DIR}/downloads" CACHE PATH "Dependencies download directory.") set(AWS_C_COMMON_URL "https://github.com/awslabs/aws-c-common.git") -set(AWS_C_COMMON_TAG "v0.4.42") +set(AWS_C_COMMON_TAG "v0.6.2") include(BuildAwsCCommon) set(AWS_CHECKSUMS_URL "https://github.com/awslabs/aws-checksums.git") @@ -26,4 +26,4 @@ set(AWS_EVENT_STREAM_TAG "v0.1.5") include(BuildAwsEventStream) -add_dependencies(AwsCEventStream AwsCCommon AwsChecksums) \ No newline at end of file +add_dependencies(AwsCEventStream AwsCCommon AwsChecksums) genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/awssdk/cjson.patch000066400000000000000000000053501453617025200245100ustar00rootroot00000000000000diff -Naur awssdk-build/aws-cpp-sdk-core/source/external/cjson/cJSON.cpp awssdk-build.modify/aws-cpp-sdk-core/source/external/cjson.modify/cJSON.cpp --- awssdk-build/aws-cpp-sdk-core/source/external/cjson/cJSON.cpp 2023-02-21 16:47:21.000000000 -0800 +++ awssdk-build/aws-cpp-sdk-core/source/external/cjson.modify/cJSON.cpp 2023-02-21 16:46:14.000000000 -0800 @@ -120,7 +120,7 @@ CJSON_PUBLIC(const char*) cJSON_Version(void) { static char version[15]; - sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); + snprintf(version, sizeof(version), "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); return version; } @@ -566,26 +566,28 @@ return false; } + size_t number_buffer_size = sizeof(number_buffer); + /* For integer which is out of the range of [INT_MIN, INT_MAX], valuestring is an integer literal. */ if (item->valuestring) { - length = sprintf((char*)number_buffer, "%s", item->valuestring); + length = snprintf((char*)number_buffer, number_buffer_size, "%s", item->valuestring); } /* This checks for NaN and Infinity */ else if (isnan(d) || isinf(d)) { - length = sprintf((char*)number_buffer, "null"); + length = snprintf((char*)number_buffer, number_buffer_size, "null"); } else { /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ - length = sprintf((char*)number_buffer, "%1.15g", d); + length = snprintf((char*)number_buffer, number_buffer_size, "%1.15g", d); /* Check whether the original double can be recovered */ if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || !compare_double((double)test, d)) { /* If not, print with 17 decimal places of precision */ - length = sprintf((char*)number_buffer, "%1.17g", d); + length = snprintf((char*)number_buffer, number_buffer_size, "%1.17g", d); } } @@ -1018,7 +1020,7 @@ break; default: /* escape and print as unicode codepoint */ - sprintf((char*)output_pointer, "u%04x", *input_pointer); + snprintf((char*)output_pointer, output_buffer->length - (output_pointer - output_buffer->buffer), "u%04x", *input_pointer); output_pointer += 4; break; } @@ -2470,7 +2472,7 @@ if (num > INT_MAX || num < INT_MIN) { char buf[21]; - sprintf(buf, "%lld", num); + snprintf(buf, sizeof(buf), "%lld", num); item->valuestring = (char*)cJSON_strdup((const unsigned char*)buf, &global_hooks); } genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/awssdk/eventstreamdecoder.patch000066400000000000000000000032701453617025200272560ustar00rootroot00000000000000diff -Naur awssdk-build.old/aws-cpp-sdk-core/source/utils/event/EventStreamDecoder.cpp awssdk-build/aws-cpp-sdk-core/source/utils/event/EventStreamDecoder.cpp --- awssdk-build.old/aws-cpp-sdk-core/source/utils/event/EventStreamDecoder.cpp 2023-02-22 11:43:33.493965856 -0800 +++ awssdk-build/aws-cpp-sdk-core/source/utils/event/EventStreamDecoder.cpp 2023-02-22 10:30:53.561549772 -0800 @@ -72,9 +72,7 @@ assert(handler); if (!handler) { - AWS_LOGSTREAM_ERROR(EVENT_STREAM_DECODER_CLASS_TAG, "Payload received, but decoder encountered internal errors before." - "ErrorCode: " << EventStreamErrorsMapper::GetNameForError(handler->GetInternalError()) << ", " - "ErrorMessage: " << handler->GetEventPayloadAsString()); + AWS_LOGSTREAM_ERROR(EVENT_STREAM_DECODER_CLASS_TAG, "Payload received, but handler is null."); return; } handler->WriteMessageEventPayload(static_cast(payload->buffer), payload->len); @@ -129,9 +127,7 @@ assert(handler); if (!handler) { - AWS_LOGSTREAM_ERROR(EVENT_STREAM_DECODER_CLASS_TAG, "Payload received, but decoder encountered internal errors before." - "ErrorCode: " << EventStreamErrorsMapper::GetNameForError(handler->GetInternalError()) << ", " - "ErrorMessage: " << handler->GetEventPayloadAsString()); + AWS_LOGSTREAM_ERROR(EVENT_STREAM_DECODER_CLASS_TAG, "Header received, but handler is null."); return; } genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/gcssdk/000077500000000000000000000000001453617025200223325ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/gcssdk/absl.patch000066400000000000000000000014501453617025200242740ustar00rootroot00000000000000diff -Naur abseil-build.old/absl/debugging/failure_signal_handler.cc abseil-build/absl/debugging/failure_signal_handler.cc > ~/TileDB/cmake/patches/absl.patch --- abseil-build.old/absl/debugging/failure_signal_handler.cc 2023-02-22 11:36:22.173323574 -0800 +++ abseil-build/absl/debugging/failure_signal_handler.cc 2023-02-22 11:03:56.348288528 -0800 @@ -135,7 +135,7 @@ #else const size_t page_mask = sysconf(_SC_PAGESIZE) - 1; #endif - size_t stack_size = (std::max(SIGSTKSZ, 65536) + page_mask) & ~page_mask; + size_t stack_size = (std::max(SIGSTKSZ, 65536) + page_mask) & ~page_mask; #if defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) // Account for sanitizer instrumentation requiring additional stack space. genomicsdb-0.0~git20231212.9d7ddd0/cmake/patches/gcssdk/gcs_ossl.patch000066400000000000000000000162401453617025200251720ustar00rootroot00000000000000--- gcssdk-build-orig/google/cloud/storage/hashing_options.cc 2021-02-02 08:32:25.000000000 +0530 +++ gcssdk-build/google/cloud/storage/hashing_options.cc 2023-03-18 12:20:11.921816224 +0530 @@ -14,9 +14,9 @@ #include "google/cloud/storage/hashing_options.h" #include "google/cloud/storage/internal/openssl_util.h" +#include "google/cloud/storage/tiledb_openssl_shim.h" #include "google/cloud/internal/big_endian.h" #include -#include #include namespace google { @@ -24,13 +24,29 @@ namespace storage { inline namespace STORAGE_CLIENT_NS { std::string ComputeMD5Hash(std::string const& payload) { - MD5_CTX md5; - MD5_Init(&md5); - MD5_Update(&md5, payload.c_str(), payload.size()); - - std::string hash(MD5_DIGEST_LENGTH, ' '); - MD5_Final(reinterpret_cast(&hash[0]), &md5); - return internal::Base64Encode(hash); + if (OpenSSL_version_num() < 0x30000000L) { + MD5_CTX md5; + MD5_Init(&md5); + MD5_Update(&md5, payload.c_str(), payload.size()); + + std::string hash(MD5_DIGEST_LENGTH, ' '); + MD5_Final(reinterpret_cast(&hash[0]), &md5); + return internal::Base64Encode(hash); + + } else { + EVP_MD_CTX* mdctx; + unsigned int md5_digest_len = 0; + md5_digest_len = EVP_MD_get_size(EVP_sha1()); + unsigned char md5_digest[md5_digest_len]; + + mdctx = EVP_MD_CTX_new(); + EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); + + EVP_DigestUpdate(mdctx, payload.c_str(), payload.size()); + EVP_DigestFinal_ex(mdctx, md5_digest, &md5_digest_len); + EVP_MD_CTX_free(mdctx); + return internal::Base64Encode(std::string((char*)md5_digest)); + } } std::string ComputeCrc32cChecksum(std::string const& payload) { --- gcssdk-build-orig/google/cloud/storage/internal/hash_validator_impl.h 2021-02-02 08:32:25.000000000 +0530 +++ gcssdk-build/google/cloud/storage/internal/hash_validator_impl.h 2023-03-16 23:21:07.685891559 +0530 @@ -16,8 +16,8 @@ #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_HASH_VALIDATOR_IMPL_H #include "google/cloud/storage/internal/hash_validator.h" +#include "google/cloud/storage/tiledb_openssl_shim.h" #include "google/cloud/storage/version.h" -#include namespace google { namespace cloud { @@ -43,6 +43,7 @@ private: MD5_CTX context_; + EVP_MD_CTX* evp_context_; std::string received_hash_; }; --- gcssdk-build-orig/google/cloud/storage/internal/hash_validator_impl.cc 2021-02-02 08:32:25.000000000 +0530 +++ gcssdk-build/google/cloud/storage/internal/hash_validator_impl.cc 2023-03-18 12:24:56.367288761 +0530 @@ -15,6 +15,7 @@ #include "google/cloud/storage/internal/hash_validator_impl.h" #include "google/cloud/storage/internal/openssl_util.h" #include "google/cloud/storage/object_metadata.h" +#include "google/cloud/storage/tiledb_openssl_shim.h" #include "google/cloud/internal/big_endian.h" #include @@ -23,10 +24,21 @@ namespace storage { inline namespace STORAGE_CLIENT_NS { namespace internal { -MD5HashValidator::MD5HashValidator() : context_{} { MD5_Init(&context_); } +MD5HashValidator::MD5HashValidator() : context_{} { + if (OpenSSL_version_num() < 0x30000000L) { + MD5_Init(&context_); + } else { + evp_context_ = EVP_MD_CTX_new(); + EVP_DigestInit_ex(evp_context_, EVP_md5(), NULL); + } +} void MD5HashValidator::Update(char const* buf, std::size_t n) { - MD5_Update(&context_, buf, n); + if (OpenSSL_version_num() < 0x30000000L) { + MD5_Update(&context_, buf, n); + } else { + EVP_DigestUpdate(evp_context_, buf, n); + } } void MD5HashValidator::ProcessMetadata(ObjectMetadata const& meta) { @@ -59,11 +71,23 @@ } HashValidator::Result MD5HashValidator::Finish() && { - std::string hash(MD5_DIGEST_LENGTH, ' '); - MD5_Final(reinterpret_cast(&hash[0]), &context_); - auto computed = Base64Encode(hash); - bool is_mismatch = !received_hash_.empty() && (received_hash_ != computed); - return Result{std::move(received_hash_), std::move(computed), is_mismatch}; + if (OpenSSL_version_num() < 0x30000000L) { + std::string hash(MD5_DIGEST_LENGTH, ' '); + MD5_Final(reinterpret_cast(&hash[0]), &context_); + auto computed = Base64Encode(hash); + bool is_mismatch = !received_hash_.empty() && (received_hash_ != computed); + return Result{std::move(received_hash_), std::move(computed), is_mismatch}; + } else { + unsigned int md5_digest_len = 0; + md5_digest_len = EVP_MD_get_size(EVP_sha1()); + unsigned char md5_digest[md5_digest_len]; + + EVP_DigestFinal_ex(evp_context_, md5_digest, &md5_digest_len); + auto computed = Base64Encode(std::string((char*)md5_digest)); + bool is_mismatch = !received_hash_.empty() && (received_hash_ != computed); + EVP_MD_CTX_free(evp_context_); + return Result{std::move(received_hash_), std::move(computed), is_mismatch}; + } } void Crc32cHashValidator::Update(char const* buf, std::size_t n) { --- gcssdk-build-orig/google/cloud/storage/internal/sha256_hash.cc 2021-02-02 08:32:25.000000000 +0530 +++ gcssdk-build/google/cloud/storage/internal/sha256_hash.cc 2023-03-18 12:26:05.229536433 +0530 @@ -13,10 +13,10 @@ // limitations under the License. #include "google/cloud/storage/internal/sha256_hash.h" -#include -#include +#include "google/cloud/storage/tiledb_openssl_shim.h" #include +#define SHA256_DIGEST_LENGTH 32 namespace google { namespace cloud { namespace storage { @@ -27,17 +27,32 @@ template ::type = 0> std::vector Sha256Hash(Byte const* data, std::size_t count) { - SHA256_CTX sha256; - SHA256_Init(&sha256); - SHA256_Update(&sha256, data, count); - - std::array hash{}; - SHA256_Final(hash.data(), &sha256); - // Note that this constructor (from a range) converts the `unsigned char` to - // `std::uint8_t` if needed, this should work because (a) the values returned - // by `SHA256_Final()` are 8-bit values, and (b) because if `std::uint8_t` - // exists it must be large enough to fit an `unsigned char`. - return {hash.begin(), hash.end()}; + if (OpenSSL_version_num() < 0x30000000L) { + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, data, count); + + std::array hash{}; + SHA256_Final(hash.data(), &sha256); + // Note that this constructor (from a range) converts the `unsigned char` to + // `std::uint8_t` if needed, this should work because (a) the values + // returned by `SHA256_Final()` are 8-bit values, and (b) because if + // `std::uint8_t` exists it must be large enough to fit an `unsigned char`. + return {hash.begin(), hash.end()}; + } else { + EVP_MD_CTX* mdctx = nullptr; + mdctx = EVP_MD_CTX_new(); + EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL); + EVP_DigestUpdate(mdctx, data, count); + + unsigned int md5_digest_len = 0; + md5_digest_len = EVP_MD_get_size(EVP_sha1()); + unsigned char md5_digest[md5_digest_len]; + + EVP_DigestFinal_ex(mdctx, md5_digest, &md5_digest_len); + EVP_MD_CTX_free(mdctx); + return {md5_digest, md5_digest + md5_digest_len}; + } } } // namespace genomicsdb-0.0~git20231212.9d7ddd0/core/000077500000000000000000000000001453617025200172755ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/CMakeLists.txt000066400000000000000000000064641453617025200220470ustar00rootroot00000000000000# # core/CMakeLists.txt # # # The MIT License # # Copyright (c) 2016 MIT and Intel Corporation # Copyright (vP 2023 Omics Data Automation, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Include core header directories file(GLOB TILEDB_CORE_INCLUDE_DIRS "include/*" "src/codec/external/lz4" "src/codec/external/bitshuffle") include_directories(${TILEDB_CORE_INCLUDE_DIRS}) # Gather the core source files file(GLOB_RECURSE TILEDB_CORE_SOURCES "src/*.cc" "src/codec/external/bitshuffle/*.c") if(LZ4_FOUND) Message(STATUS "Using LZ4 from ${LZ4_LIBRARIES}") else() list(APPEND TILEDB_CORE_SOURCES "src/codec/external/lz4/lz4.c") endif() # Compile sources with PIC add_library(TILEDB_CORE_OBJECTS OBJECT ${TILEDB_CORE_SOURCES}) set_property(TARGET TILEDB_CORE_OBJECTS PROPERTY POSITION_INDEPENDENT_CODE ON) add_dependencies(TILEDB_CORE_OBJECTS awssdk-ep) add_dependencies(TILEDB_CORE_OBJECTS gcssdk-ep) # Create static and shared libraries if(USE_HDFS) add_library(tiledb_static STATIC $ $) add_library(tiledb_shared SHARED $ $) else() add_library(tiledb_static STATIC $) add_library(tiledb_shared SHARED $) endif() set_target_properties(tiledb_static tiledb_shared PROPERTIES OUTPUT_NAME "tiledb") # See https://glandium.org/blog/?p=2764 for workaound for undefined weak symbols in MacOS if(APPLE) set(TILEDB_LINK_OPTIONS "-Wl,-U,_EVP_MAC_CTX_free;-Wl,-U,_EVP_MAC_CTX_new;-Wl,-U,_EVP_MAC_fetch;;-Wl,-U,_EVP_MAC_final;-Wl,-U,_EVP_MAC_free;-Wl,-U,_EVP_MAC_init;-Wl,-U,_EVP_MAC_update;-Wl,-U,_EVP_MD_get_size;-Wl,-U,_OSSL_PARAM_construct_end;-Wl,-U,_OSSL_PARAM_construct_utf8_string") target_link_options(tiledb_static PUBLIC ${TILEDB_LINK_OPTIONS}) target_link_options(tiledb_shared PUBLIC ${TILEDB_LINK_OPTIONS}) endif() target_link_libraries(tiledb_static muparserx azure-storage-adls ${AWSSDK_LINK_LIBRARIES} ${GCSSDK_LINK_LIBRARIES}) target_link_libraries(tiledb_shared ${TILEDB_LIB_DEPENDENCIES}) # Install libraries install( TARGETS tiledb_static tiledb_shared LIBRARY DESTINATION lib ARCHIVE DESTINATION lib ) file(GLOB TILEDB_HEADERS "include/c_api/*.h") install(FILES ${TILEDB_HEADERS} DESTINATION include) genomicsdb-0.0~git20231212.9d7ddd0/core/EnvironmentVariables.md000066400000000000000000000023111453617025200237510ustar00rootroot00000000000000## Environment Variables The use of these Environment Variables will alter the behavior of TileDB. These are meant to be experimental and may be removed in future when the functionality is main streamed. * TILEDB_DISABLE_FILE_LOCKING Relevant only for PosixFS. No read/write locks are maintained for arrays. If used, it is the responsibility of the client to ensure that creating/updating/deleting arrays and array fragments are done with utmost care. * TILEDB_KEEP_FILE_HANDLES_OPEN Relevant only for PosixFS. All file handles during writes are kept open until a PosixFS::close_file() is called. * TILEDB_UPLOAD_BUFFER_SIZE Helps write out buffered array fragments to the datastore. If this is set to 0(default for PosixFS and HDFS), array fragments are written out immediately. * TILEDB_DOWNLOAD_BUFFER_SIZE Helps prefetch/read from buffered array fragments from the datastore. If this is set to 0(default for PosixFS and HDFS), array fragments are read unbuffered. * TILEDB_MAX_STREAM_SIZE For azure blob storage, use download_blob_to_stream to read lengths < TILEDB_MAX_STREAM_SIZE. If this is not set, the default is 1024 bytes defined in core/include/storage_manager/storage_azure_blob.h. genomicsdb-0.0~git20231212.9d7ddd0/core/include/000077500000000000000000000000001453617025200207205ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/000077500000000000000000000000001453617025200220365ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/aio_request.h000066400000000000000000000075321453617025200245360ustar00rootroot00000000000000/** * @file aio_request.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file declares the AIO_Request struct. */ #ifndef __AIO_REQUEST_H__ #define __AIO_REQUEST_H__ #include /** Describes an AIO (read or write) request. */ struct AIO_Request { /** * An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * array initialization or when resetting the attributes. The case of * variable-sized attributes is special. Instead of providing a single * buffer for such an attribute, **two** must be provided: the second * will hold the variable-sized cell values, whereas the first holds the * start offsets of each cell in the second buffer. */ void** buffers_; /** * The sizes (in bytes) allocated by the user for the input * buffers (there is a one-to-one correspondence). The function will attempt * to write as many results as can fit in the buffers, and potentially * alter the buffer size to indicate the size of the *useful* data written * in the buffer. */ size_t* buffer_sizes_; /** Function to be called upon completion of the request. */ void *(*completion_handle_) (void*); /** Data to be passed to the completion handle. */ void* completion_data_; /** A unique request id. */ size_t id_; /** * It can be one of the following: * - TILEDB_ARRAY_READ * - TILEDB_ARRAY_READ_SORTED_COL * - TILEDB_ARRAY_READ_SORTED_ROW * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_UNSORTED */ int mode_; /** * Applicable only to read requests. * Indicates whether a buffer has overflowed during a read request. * If it is NULL, it will be ignored. Otherwise, it must be an array * with as many elements as the number of buffers above. */ bool* overflow_; /** * The status of the AIO request. It can be one of the following: * - TILEDB_AIO_COMPLETED * The request is completed. * - TILEDB_AIO_INPROGRESS * The request is still in progress. * - TILEDB_AIO_OVERFLOW * At least one of the input buffers overflowed (applicable only to AIO * read requests) * - TILEDB_AIO_ERR * The request caused an error (and thus was canceled). */ int* status_; /** * The subarray in which the array read/write will be * constrained on. It should be a sequence of [low, high] pairs (one * pair per dimension), whose type should be the same as that of the * coordinates. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. */ const void* subarray_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array.h000066400000000000000000000617401453617025200233350ustar00rootroot00000000000000/** * @file array.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class Array. */ #ifndef __ARRAY_H__ #define __ARRAY_H__ #include "aio_request.h" #include "array_read_state.h" #include "array_sorted_read_state.h" #include "array_sorted_write_state.h" #include "array_schema.h" #include "book_keeping.h" #include "expression.h" #include "fragment.h" #include "storage_manager_config.h" #include "tiledb_constants.h" #include "expression.h" #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_AR_OK 0 #define TILEDB_AR_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_AR_ERRMSG std::string("[TileDB::Array] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_ar_errmsg; class ArrayReadState; class ArraySortedReadState; class ArraySortedWriteState; class Fragment; /** Manages a TileDB array object. */ class Array { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ Array(); /** Destructor. */ ~Array(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** * Enters an indefinite loop that handles all the AIO requests. This is * executed in the background by the AIO thread. * * @return void. */ void aio_handle_requests(); /** * Submits an asynchronous (AIO) read request and immediately returns control * to the caller. The request is queued up and executed in the background by * another thread. * * @param aio_request The AIO read request. * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int aio_read(AIO_Request* aio_request); /** * Submits an asynchronous (AIO) write request and immediately returns control * to the caller. The request is queued up and executed in the background by * another thread. * * @param aio_request The AIO write request. * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int aio_write(AIO_Request* aio_request); /** Returns the array schema. */ const ArraySchema* array_schema() const; /** Returns the array clone. */ Array* array_clone() const; /** Returns the ids of the attributes the array focuses on. */ const std::vector& attribute_ids() const; /** Returns the configuration parameters. */ const StorageManagerConfig* config() const; /** Returns the number of fragments in this array. */ int fragment_num() const; /** Returns the fragment objects of this array. */ std::vector fragments() const; /** Returns the array mode. */ int mode() const; /** * Checks if *at least one* attribute buffer has overflown during a read * operation. * * @return *true* if at least one attribute buffer has overflown and *false* * otherwise. */ bool overflow() const; /** * Checks if an attribute buffer has overflown during a read operation. * * @param attribute_id The id of the attribute that is being checked. * @return *true* if the attribute buffer has overflown and *false* otherwise. */ bool overflow(int attribute_id) const; /** * Performs a read operation in an array, which must be initialized in read * mode. The function retrieves the result cells that lie inside * the subarray specified in init() or reset_subarray(). The results are * written in input buffers provided by the user, which are also allocated by * the user. Note that the results are written in the buffers in the same * order as that specified by the user in the init() function. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second will hold the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). The function will * attempt to write as many results as can fit in the buffers, and * potentially alter the buffer size to indicate the size of the *useful* * data written in the buffer. If a buffer cannot hold all results, the * function will still succeed, writing as much data as it can and turning * on an overflow flag which can be checked with function overflow(). The * next invocation will resume for the point the previous one stopped, * without inflicting a considerable performance penalty due to overflow. * @param skip_counts Number of cells to skip before reading data into buffer. * This can be NULL (no skip). If non NULL, the number of entries in skip_counts * must be equal to the number of entries in buffer_sizes * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int read(void** buffers, size_t* buffer_sizes, size_t* skip_counts=0); /** * Evaluates the cell based on the filter expression applied to the array. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second will hold the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). * @param positions The position of the cell in the buffer to be evaluated. * There should be one position for each of the input buffers. * @return TILEDB_AR_OK for successful evaluation and TILEDB_AR_ERR otherwise. * The onus is on the client to check if there was an error during * evaluation when TILEB_AR_ERR is returned. */ int evaluate_cell(void** buffer, size_t* buffer_sizes, int64_t* positions); /** * Performs a read operation in an array, which must be initialized in read * mode. The function retrieves the result cells that lie inside * the subarray specified in init() or reset_subarray(). The results are * written in input buffers provided by the user, which are also allocated by * the user. Note that the results are written in the buffers in the same * order they appear on the disk, which leads to maximum performance. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second will hold the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). The function will * attempt to write as many results as can fit in the buffers, and * potentially alter the buffer size to indicate the size of the *useful* * data written in the buffer. If a buffer cannot hold all results, the * function will still succeed, writing as much data as it can and turning * on an overflow flag which can be checked with function overflow(). The * next invocation will resume for the point the previous one stopped, * without inflicting a considerable performance penalty due to overflow. * @param skip_counts Number of cells to skip before reading data into buffer. * This can be NULL (no skip). If non NULL, the number of entries in skip_counts * must be equal to the number of entries in buffer_sizes * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int read_default(void** buffers, size_t* buffer_sizes, size_t* skip_counts=0); /** Returns true if the array is in read mode. */ bool read_mode() const; /** Returns the subarray in which the array is constrained. */ const void* subarray() const; /** Returns true if the array is in write mode. */ bool write_mode() const; /** Return true if the array is in consolidate mode */ bool consolidate_mode() const; /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Consolidates all fragments into a new single one, on a per-attribute basis. * Returns the new fragment (which has to be finalized outside this function), * along with the names of the old (consolidated) fragments (which also have * to be deleted outside this function). * * @param new_fragment The new fragment to be returned. * @param old_fragment_names The names of the old fragments to be returned. * @param buffer_size (Optional) The size of buffers for reading/writing attributes during consolidation. Default is 10M. * @param batch size (Optional) When specified, consolidation will occur batch-wise with a smaller batch_size set of * fragments getting consolidating together. Default is all fragments. * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int consolidate( Fragment*& new_fragment, std::vector& old_fragment_names, size_t buffer_size = TILEDB_CONSOLIDATION_BUFFER_SIZE, int batch_size = -1); /** * Consolidates a batch of fragments into a new single one, focusing on a specific attribute. * * @param new_fragment The new consolidated fragment object. * @param attribute_id The id of the target attribute. * @param buffers Array of buffers, preallocated only for specified attribute. Buffers for other attributes can be NULL * @param buffer_sizes Size of buffer allocations in buffers. Buffer sizes for all attributes except the one * specified can be 0. * @param buffer_size */ int consolidate( Fragment* new_fragment, int attribute_id, void **buffers, size_t *buffer_sizes, size_t buffer_size); /** * Finalizes the array, properly freeing up memory space. * * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int finalize(); /** * Initializes a TileDB array object. * * @param array_schema The array schema. * @param array_path_used The path to the array as specified by the caller - might be different from the one in array_schema * @param fragment_names The names of the fragments of the array. * @param book_keeping The book-keeping structures of the fragments * of the array. * @param mode The mode of the array. It must be one of the following: * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_SORTED_COL * - TILEDB_ARRAY_WRITE_SORTED_ROW * - TILEDB_ARRAY_WRITE_UNSORTED * - TILEDB_ARRAY_READ * - TILEDB_ARRAY_READ_SORTED_COL * - TILEDB_ARRAY_READ_SORTED_ROW * @param attributes A subset of the array attributes the read/write will be * constrained on. A NULL value indicates **all** attributes (including * the coordinates in the case of sparse arrays). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param subarray The subarray in which the array read/write will be * constrained on. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. * @param config Configuration parameters. * @param array_clone An clone of this array object. Used specifically in * asynchronous IO (AIO) read/write operations. * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int init( const ArraySchema* array_schema, const std::string array_path_used, const std::vector& fragment_names, const std::vector& book_keeping, int mode, const char** attributes, int attribute_num, const void* subarray, const StorageManagerConfig* config, Array* array_clone = NULL); /** * Applies a filter expression to constrain the results returned by * a read operation. * * @param filter_expression An expression string that evaluates to a boolean * to allow for cells to be filtered out from the buffers while reading. * If NULL, there is no filter applied. * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int apply_filter(const char* filter_expression); /** * Resets the attributes used upon initialization of the array. * * @param attributes The new attributes to focus on. If it is NULL, then * all the attributes are used (including the coordinates in the case of * sparse arrays). * @param attribute_num The number of the attributes. If *attributes* is NULL, * then this should be 0. * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int reset_attributes(const char** attributes, int attribute_num); /** * Resets the subarray used upon initialization of the array. This is useful * when the array is used for reading, and the user wishes to change the * query subarray without having to finalize and re-initialize the array * with a different subarray. * * @param subarray The new subarray. Note that the type of the values in * *subarray* should match the coordinates type in the array schema. * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int reset_subarray(const void* subarray); /** * Same as reset_subarray(), with the difference that the * ArraySortedReadState object of the array is not re-initialized. * * @param subarray The new subarray. Note that the type of the values in * *subarray* should match the coordinates type in the array schema. * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int reset_subarray_soft(const void* subarray); /** * Syncs all currently written files in the input array. * * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int sync(); /** * Syncs the currently written files associated with the input attribute * in the input array. * * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int sync_attribute(const std::string& attribute); /** * Performs a write operation in the array. The cell values are provided * in a set of buffers (one per attribute specified upon initialization). * Note that there must be a one-to-one correspondance between the cell * values across the attribute buffers. * * The array must be initialized in one of the following write modes, * each of which having a different behaviour: * - TILEDB_ARRAY_WRITE: \n * In this mode, the cell values are provided in the buffers respecting * the cell order on the disk. It is practically an **append** operation, * where the provided cell values are simply written at the end of * their corresponding attribute files. This mode leads to the best * performance. The user may invoke this function an arbitrary number * of times, and all the writes will occur in the same fragment. * Moreover, the buffers need not be synchronized, i.e., some buffers * may have more cells than others when the function is invoked. * - TILEDB_ARRAY_WRITE_SORTED_COL: \n * In this mode, the cell values are provided in the buffer in * column-major * order with respect to the subarray used upon array initialization. * TileDB will properly re-organize the cells so that they follow the * array cell order for storage on the disk. * - TILEDB_ARRAY_WRITE_SORTED_ROW: \n * In this mode, the cell values are provided in the buffer in row-major * order with respect to the subarray used upon array initialization. * TileDB will properly re-organize the cells so that they follow the * array cell order for storage on the disk. * - TILEDB_ARRAY_WRITE_UNSORTED: \n * This mode is applicable to sparse arrays, or when writing sparse * updates to a dense array. One of the buffers holds the coordinates. * The cells in this mode are given in an arbitrary, unsorted order * (i.e., without respecting how the cells must be stored on the disk * according to the array schema definition). Each invocation of this * function internally sorts the cells and writes them to the disk on the * proper order. In addition, each invocation creates a **new** fragment. * Finally, the buffers in each invocation must be synced, i.e., they * must have the same number of cell values across all attributes. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second holds the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int write(const void** buffers, const size_t* buffer_sizes); /** * Performs a write operation in the array. The cell values are provided * in a set of buffers (one per attribute specified upon initialization). * Note that there must be a one-to-one correspondance between the cell * values across the attribute buffers. * * The array must be initialized in moder TILEDB_ARRAY_WRITE or * TILEDB_ARRAY_WRITE_UNSORTED. These modes are essentially the default modes. * Modes TILEDB_ARRAY_WRITE_SORTED_COL and TILEDB_ARRAY_WRITE_SORTED_ROW are * more complicated and, thus, handled by the ArraySortedWriteState class. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second holds the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int write_default(const void** buffers, const size_t* buffer_sizes); /* * Free array_schema_ */ void free_array_schema(); /** * Get path used to open array - might be different from the one in the * schema if the array is moved */ const std::string& get_array_path_used() const; private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The AIO mutex condition. */ pthread_cond_t aio_cond_; /** Stores the id of the last handled AIO request. */ size_t aio_last_handled_request_; /** The AIO mutex. */ pthread_mutex_t aio_mtx_; /** The queue that stores the pending AIO requests. */ std::queue aio_queue_; /** The thread that handles all the AIO reads and writes in the background. */ pthread_t aio_thread_; /** Indicates whether the AIO thread was canceled or not. */ bool aio_thread_canceled_; /** Indicates whether the AIO thread was created or not. */ volatile bool aio_thread_created_; /** An array clone, used in AIO requests. */ Array* array_clone_; /** The array schema. */ const ArraySchema* array_schema_; /** The read state of the array. */ ArrayReadState* array_read_state_; /** The sorted read state of the array. */ ArraySortedReadState* array_sorted_read_state_; /** The sorted write state of the array. */ ArraySortedWriteState* array_sorted_write_state_; /** * The ids of the attributes the array is initialized with. Note that the * array may be initialized with a subset of attributes when writing or * reading. */ std::vector attribute_ids_; /** Configuration parameters. */ const StorageManagerConfig* config_; /** Fragment names at initialization for consolidation */ std::vector fragment_names_; /** The array fragments. */ std::vector fragments_; /** * The array mode. It must be one of the following: * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_SORTED_COL * - TILEDB_ARRAY_WRITE_SORTED_ROW * - TILEDB_ARRAY_WRITE_UNSORTED * - TILEDB_ARRAY_READ * - TILEDB_ARRAY_READ_SORTED_COL * - TILEDB_ARRAY_READ_SORTED_ROW */ int mode_; /** * The subarray in which the array is constrained. Note that the type of the * range must be the same as the type of the array coordinates. */ void* subarray_; /** * The expression object which will be used to filter values */ Expression* expression_; std::string array_path_used_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Handles an AIO request. * * @param aio_request The AIO request. The function will resolve whether it is * a read or write request based on the array mode. * @return void. * */ void aio_handle_next_request(AIO_Request* aio_request); /** * Function called by the AIO thread. * * @param context This is practically the Array object for which the function * is called (typically *this* is passed to this argument by the caller). */ static void *aio_handler(void* context); /** * Pusghes an AIO request into the AIO queue. * * @param aio_request The AIO request. * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int aio_push_request(AIO_Request* aio_request); /** * Creates the AIO thread. * * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int aio_thread_create(); /** * Destroys the AIO thread. * * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int aio_thread_destroy(); /** * Returns a new fragment name, which is in the form:
* .__MAC-address_thread-id_timestamp. For instance, * __00332a0b8c6426153_1458759561320 * * Note that this is a temporary name, initiated by a new write process. * After the new fragmemt is finalized, the array will change its name * by removing the leading '.' character. * * @return A new special fragment name on success, or "" (empty string) on * error. */ std::string new_fragment_name() const; /** * Opens the existing fragments. * * @param fragment_names The vector with the fragment names. * @param book_keeping The book-keeping of the array fragments. * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int open_fragments( const std::vector& fragment_names, const std::vector& book_keeping); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array_iterator.h000066400000000000000000000162701453617025200252440ustar00rootroot00000000000000/** * @file array_iterator.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2019 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ArrayIterator. */ #ifndef __ARRAY_ITERATOR_H__ #define __ARRAY_ITERATOR_H__ #include "array.h" #include "expression.h" /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_AIT_OK 0 #define TILEDB_AIT_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_AIT_ERRMSG std::string("[TileDB::ArrayIterator] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_ait_errmsg; /** Enables iteration (read) over an array's cells. */ class ArrayIterator { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ ArrayIterator(); /** Destructor. */ ~ArrayIterator(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Return the array name. */ const std::string& array_name() const; /** * Checks if the the iterator has reached its end. * * @return *true* if the iterator has reached its end and *false* otherwise. */ bool end() const; /** * Retrieves the current cell value for a particular attribute. * * @param attribute_id The id of the attribute for which the cell value * is retrieved. This id corresponds to the position of the attribute name * placed in the *attributes* input of Array::init() with which the array * was initialized. If *attributes* was NULL in the above function, then * the attribute id corresponds to the order in which the attributes were * defined in the array schema upon the array creation. Note that, in that * case, the extra coordinates attribute corresponds to the last extra * attribute, i.e., its id is *attribute_num*. * @param value The cell value to be retrieved. Note that its type is the * same as that defined in the array schema. * @param value_size The size (in bytes) of the retrieved value. * @return TILEDB_AIT_OK on success, and TILEDB_AIT_ERR on error. */ int get_value(int attribute_id, const void** value, size_t* value_size) const; /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Initializes an iterator for reading cells from an initialized array. * * @param array The array the iterator is initialized for. * @param buffers This is an array of buffers similar to Array::read(). * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read cells. The iterator will * read from the disk the relevant cells in batches, by fitting as many * cell values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding sizes (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many cells as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @param filter_expression An expression string that evaluates to a boolean * to allow for cells to be filtered out from the buffers while reading. * If empty, no filter is applied. * @return TILEDB_AIT_OK on success, and TILEDB_AIT_ERR on error. */ int init( Array* array, void** buffers, size_t* buffer_sizes, const char* filter_expression = NULL); /** * Resets the subarray used upon initialization of the iterator. This is useful * when the array is used for reading, and the user wishes to change the * query subarray without having to finalize and re-initialize the array * with a different subarray. * * @param subarray The new subarray. Note that the type of the values in * *subarray* should match the coordinates type in the array schema. * If not null, array_->reset_subarray() is called * @return TILEDB_AIT_OK on success, and TILEDB_AIT_ERR on error. */ int reset_subarray(const void* subarray); /** * Finalizes the array iterator, properly freeing the allocating memory space. * * @return TILEDB_AIT_OK on success, and TILEDB_AIT_ERR on error. */ int finalize(); /** * Advances the iterator by one cell. * * @return TILEDB_AIT_OK on success, and TILEDB_AIT_ERR on error. */ int next(); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The array the iterator works on. */ Array* array_; /** The buffers used for prefetching. */ void** buffers_; /** * The corresponding sizes of *buffers*. These will be likely changed upon * each prefetching. */ size_t* buffer_sizes_; /** * The original corresponding sizes of *buffers*, as they were passed in * init(). */ std::vector buffer_allocated_sizes_; /** A flag indicating whether the iterator has reached its end. */ bool end_; /** * The position of the current value the iterator is on in the currently * buffered cells in *buffers*. A separate position is maintained for each * attribute buffer. */ std::vector pos_; /** The number of cells currently in *buffers* (one per attribute). */ std::vector cell_num_; /** Stores the cell size of each attribute. */ std::vector cell_sizes_; /** The index of the *buffers* element each attribute corresponds to. */ std::vector buffer_i_; /** Number of variable attributes. */ int var_attribute_num_; Expression* expression_ = NULL; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array_read_state.h000066400000000000000000000736121453617025200255310ustar00rootroot00000000000000/** * @file array_read_state.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ArrayReadState. */ #ifndef __ARRAY_READ_STATE_H__ #define __ARRAY_READ_STATE_H__ #include "array.h" #include "array_schema.h" #define __STDC_FORMAT_MACROS #include #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_ARS_OK 0 #define TILEDB_ARS_ERR -1 /**@}*/ /** Size of the starting offset of a variable cell value. */ #define TILEDB_CELL_VAR_OFFSET_SIZE sizeof(size_t) /** Default error message. */ #define TILEDB_ARS_ERRMSG std::string("[TileDB::ArrayReadState] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_ars_errmsg; class Array; class ReadState; /** Stores the state necessary when reading cells from the array fragments. */ class ArrayReadState { public: /* ********************************* */ /* TYPE DEFINITIONS */ /* ********************************* */ /** * Class of fragment cell range objects used in the priority queue algorithm. */ template class PQFragmentCellRange; /** * Wrapper of comparison function in the priority queue of the fragment cell * ranges. */ template class SmallerPQFragmentCellRange; /** A cell position pair [first, second]. */ typedef std::pair CellPosRange; /** A pair [fragment_id, tile_pos]. */ typedef std::pair FragmentInfo; /** A pair of fragment info and fragment cell position range. */ typedef std::pair FragmentCellPosRange; /** A vector of fragment cell posiiton ranges. */ typedef std::vector FragmentCellPosRanges; /** A vector of vectors of fragment cell position ranges. */ typedef std::vector FragmentCellPosRangesVec; /** * A pair of fragment info and cell range, where the cell range is defined * by two bounding coordinates. */ typedef std::pair FragmentCellRange; /** A vector of fragment cell ranges. */ typedef std::vector FragmentCellRanges; /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param array The array this array read state belongs to. */ ArrayReadState(const Array* array); /** Destructor. */ ~ArrayReadState(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Indicates whether the read on at least one attribute overflowed. */ bool overflow() const; /** Indicates whether the read on a particular attribute overflowed. */ bool overflow(int attribute_id) const; /** * Performs a read operation in an array, which must be initialized with mode * TILEDB_ARRAY_READ. The function retrieves the result cells that lie inside * the subarray specified in Array::init() or Array::reset_subarray(). The * results are written in input buffers provided by the user, which are also * allocated by the user. Note that the results are written in the buffers in * the same order they appear on the disk, which leads to maximum performance. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * Array::init() or Array::reset_attributes(). The case of variable-sized * attributes is special. Instead of providing a single buffer for such an * attribute, **two** must be provided: the second will hold the * variable-sized cell values, whereas the first holds the start offsets * of each cell in the second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). The function will * attempt to write as many results as can fit in the buffers, and * potentially alter the buffer size to indicate the size of the *useful* * data written in the buffer. If a buffer cannot hold all results, the * function will still succeed, writing as much data as it can and turning * on an overflow flag which can be checked with function overflow(). The * next invocation will resume for the point the previous one stopped, * without inflicting a considerable performance penalty due to overflow. * @param skip_counts Number of cells to skip before reading data into buffer. * This can be NULL (no skip). If non NULL, the number of entries in skip_counts * must be equal to the number of entries in buffer_sizes * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read(void** buffers, size_t* buffer_sizes, size_t* skip_counts=0); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The array this array read state belongs to. */ const Array* array_; /** The array schema. */ const ArraySchema* array_schema_; /** The number of array attributes. */ int attribute_num_; /** The size of the array coordinates. */ size_t coords_size_; /** Indicates whether the read operation for this query is done. */ bool done_; /** State per attribute indicating the number of empty cells written. */ std::vector empty_cells_written_; /** * The bounding coordinates of the current tiles for all fragments. Applicable * only to the **sparse** array case. */ std::vector fragment_bounding_coords_; /** Holds the fragment cell positions ranges of all active read rounds. */ FragmentCellPosRangesVec fragment_cell_pos_ranges_vec_; /** Practically records which read round each attribute is on. */ std::vector fragment_cell_pos_ranges_vec_pos_; /** Number of array fragments. */ int fragment_num_; /** Stores the read state of each fragment. */ std::vector fragment_read_states_; /** * The minimum bounding coordinates end point. Applicable only to the * **sparse** array case. */ void* min_bounding_coords_end_; /** Indicates overflow for each attribute. */ std::vector overflow_; /** Indicates whether the current read round is done for each attribute. */ std::vector read_round_done_; /** The current tile coordinates of the query subarray. */ void* subarray_tile_coords_; /** The tile domain of the query subarray. */ void* subarray_tile_domain_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** Cleans fragment cell positions that are processed by all attributes. */ void clean_up_processed_fragment_cell_pos_ranges(); /** * Computes the cell position ranges that must be copied from each fragment to * the user buffers for the current read round. The cell positions are * practically the relative positions of the cells in their tile on the * disk. The function properly cleans up the input fragment cell ranges. * * @tparam T The coordinates type. * @param fragment_cell_ranges The input fragment cell ranges. * @param fragment_cell_pos_ranges The output fragment cell position ranges. * @return TILEDB_ARS_OK on success and TILEDB_ARS_ERR on error. */ template int compute_fragment_cell_pos_ranges( FragmentCellRanges& fragment_cell_ranges, FragmentCellPosRanges& fragment_cell_pos_ranges) const; /** * Computes the smallest end bounding coordinates for the current read round. * * @tparam T The coordinates type. * @return void */ template void compute_min_bounding_coords_end(); /** * Computes the relevant fragment cell ranges for the current read run, * focusing on the **dense* array case. These cell ranges will be properly * cut and sorted later on. * * @tparam T The coordinates type. * @param unsorted_fragment_cell_ranges It will hold the result of this * function. * @return TILEDB_ARS_OK on success and TILEDB_ARS_ERR on error. */ template int compute_unsorted_fragment_cell_ranges_dense( std::vector& unsorted_fragment_cell_ranges); /** * Computes the relevant fragment cell ranges for the current read run, * focusing on the **sparse* array case. These cell ranges will be properly * cut and sorted later on. This function also properly updates the start * bounding coordinates of the active tiles (to exceed the minimum bounding * coordinates end). * * @tparam T The coordinates type. * @param unsorted_fragment_cell_ranges It will hold the result of this * function. * @return TILEDB_ARS_OK on success and TILEDB_ARS_ERR on error. */ template int compute_unsorted_fragment_cell_ranges_sparse( std::vector& unsorted_fragment_cell_ranges); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer. * * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read copy be performed into. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param remaining_skip_count The number of cells to skip before copying * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ int copy_cells( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer. * * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read copy be performed into. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ int copy_cells( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer. * * @tparam T The attribute type. * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read copy be performed into. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param remaining_skip_count The number of cells to skip before copying * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ template int copy_cells( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer, focusing on a **variable-sized** attribute. * * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read will be performed into - offsets of * cells in *buffer_var*. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param remaining_skip_count The number of cells to skip before copying * @param buffer_var The buffer where the copy will be performed into - actual * variable-sized cell values. * @param buffer_var_size The size (in bytes) of *buffer_var*. * @param buffer_var_offset The offset in *buffer_var* where the copy will * start from. * @param remaining_skip_count_var The number of cells to skip before copying for var field * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ int copy_cells_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer, focusing on a **variable-sized** attribute. * * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read will be performed into - offsets of * cells in *buffer_var*. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param buffer_var The buffer where the copy will be performed into - actual * variable-sized cell values. * @param buffer_var_size The size (in bytes) of *buffer_var*. * @param buffer_var_offset The offset in *buffer_var* where the copy will * start from. * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ int copy_cells_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer, focusing on a **variable-sized** attribute. * * @tparam T The attribute type. * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read will be performed into - offsets of * cells in *buffer_var*. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param remaining_skip_count The number of cells to skip before copying * @param buffer_var The buffer where the copy will be performed into - actual * variable-sized cell values. * @param buffer_var_size The size (in bytes) of *buffer_var*. * @param buffer_var_offset The offset in *buffer_var* where the copy will * start from. * @param remaining_skip_count The number of cells to skip before copying for var field * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ template int copy_cells_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer, filling with special empty values. * * @tparam T The attribute type. * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the copy will be performed into. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param cell_pos_range The cell range to be copied. * @param remaining_skip_count The number of cells to skip before copying * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ template void copy_cells_with_empty( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, const CellPosRange& cell_pos_range, size_t& remaining_skip_count); /** * Copies the cell ranges calculated in the current read round into the * targeted attribute buffer, feeling with special empty values, and focusing * on a **variable-sized** attribute. * * @tparam T The attribute type. * @param attribute_id The id of the targeted attribute. * @param buffer The buffer where the read will be performed into - offsets of * cells in *buffer_var*. * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param remaining_skip_count The number of cells to skip before copying * @param buffer_var The buffer where the copy will be performed into - actual * variable-sized cell values. * @param buffer_var_size The size (in bytes) of *buffer_var*. * @param buffer_var_offset The offset in *buffer_var* where the copy will * start from. * @param remaining_skip_count_var The number of cells to skip before copying for the var field * @param cell_pos_range The cell range to be copied. * @return TILEDB_ARS on success and TILEDB_ARS_ERR on error. */ template void copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var, const CellPosRange& cell_pos_range); /** * Returns a list of cell ranges accounting for the empty area in the overlap * between the subarray query and the current overlapping tile. * * @return A list of cell ranges representing empty cells. */ template FragmentCellRanges empty_fragment_cell_ranges() const; /** * Gets the next fragment cell ranges that are relevant in the current read * round, focusing on the dense case. * * @tparam T The coordinates type. * @return TILEDB_ARS_OK on success and TILEDB_ARS_ERR on error. */ template int get_next_fragment_cell_ranges_dense(); /** * Gets the next fragment cell ranges that are relevant in the current read * round, focusing on the sparse case. * * @tparam T The coordinates type. * @return TILEDB_ARS_OK on success and TILEDB_ARS_ERR on error. */ template int get_next_fragment_cell_ranges_sparse(); /** * Gets the next overlapping tiles in the fragment read states, for the case * of **dense** arrays. * * @tparam T The coordinates type. * @return void */ template void get_next_overlapping_tiles_dense(); /** * Gets the next overlapping tiles in the fragment read states, for the case * of **sparse** arrays. * * @tparam T The coordinates type. * @return void */ template void get_next_overlapping_tiles_sparse(); /** * Gets the next subarray tile coordinates inside the subarray tile domain. * * @tparam T The coordinates type. * @return void */ template void get_next_subarray_tile_coords(); /** * Initializes the tile coordinates falling in the query subarray. Applicable * only to the **dense** array case. * * @tparam T The coordinates type. * @return void */ template void init_subarray_tile_coords(); /** * Performs a read operation in a **dense** array. * * @param buffers See read(). * @param buffer_sizes See read(). * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read_dense(void** buffers, size_t* buffer_sizes); /** * Performs a read operation in a **dense** array, focusing on a single * attribute. * * @param attribute_id The attribute this read focuses on. * @param buffer See read(). * @param buffer_size See read(). * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read_dense_attr( int attribute_id, void* buffer, size_t& buffer_size); /** * Performs a read operation in a **dense** array, focusing on a single * attribute. * * @tparam T The coordinates type. * @param attribute_id The attribute this read focuses on. * @param buffer See read(). * @param buffer_size See read(). * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ template int read_dense_attr( int attribute_id, void* buffer, size_t& buffer_size); /** * Performs a read operation in a **dense** array, focusing on a single * **variable-sized** attribute. * * @param attribute_id The attribute this read focuses on. * @param buffer See read() - start offsets in *buffer_var*. * @param buffer_size See read(). * @param buffer_var See read() - actual variable-sized cell values. * @param buffer_var_size See read(). * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read_dense_attr_var( int attribute_id, void* buffer, size_t& buffer_size, void* buffer_var, size_t& buffer_var_size); /** * Performs a read operation in a **dense** array, focusing on a single * **variable-sized** attribute. * * @tparam T The coordinates type. * @param attribute_id The attribute this read focuses on. * @param buffer See read() - start offsets in *buffer_var*. * @param buffer_size See read(). * @param buffer_var See read() - actual variable-sized cell values. * @param buffer_var_size See read(). * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ template int read_dense_attr_var( int attribute_id, void* buffer, size_t& buffer_size, void* buffer_var, size_t& buffer_var_size); /** * Performs a read operation in a **sparse** array. * * @param buffers See read(). * @param buffer_sizes See read(). * @param skip_counts See read() * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read_sparse(void** buffers, size_t* buffer_sizes, size_t* skip_counts); /** * Performs a read operation in a **sparse** array, focusing on a single * attribute. * * @param attribute_id The attribute this read focuses on. * @param buffer See read(). * @param buffer_size See read(). * @param skip_count See read() * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read_sparse_attr( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count); /** * Performs a read operation in a **sparse** array, focusing on a single * attribute. * * @tparam T The coordinates type. * @param attribute_id The attribute this read focuses on. * @param buffer See read(). * @param buffer_size See read(). * @param skip_count See read() * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ template int read_sparse_attr( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count); /** * Performs a read operation in a **sparse** array, focusing on a single * **variable-sized** attribute. * * @param attribute_id The attribute this read focuses on. * @param buffer See read() - start offsets in *buffer_var*. * @param buffer_size See read(). * @param skip_count See read() * @param buffer_var See read() - actual variable-sized cell values. * @param buffer_var_size See read(). * @param skip_count_var See read() * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ int read_sparse_attr_var( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count, void* buffer_var, size_t& buffer_var_size, size_t& skip_count_var); /** * Performs a read operation in a **sparse** array, focusing on a single * **variable-sized** attribute. * * @tparam T Coordinates type. * @param attribute_id The attribute this read focuses on. * @param buffer See read() - start offsets in *buffer_var*. * @param buffer_size See read(). * @param skip_count See read() * @param buffer_var See read() - actual variable-sized cell values. * @param buffer_var_size See read(). * @param skip_count_var See read() * @return TILEDB_ARS_OK for success and TILEDB_ARS_ERR for error. */ template int read_sparse_attr_var( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count, void* buffer_var, size_t& buffer_var_size, size_t& skip_count_var); /** * Uses the heap algorithm to cut and sort the relevant cell ranges for * the current read run. The function properly cleans up the input * unsorted fragment cell ranges. * * @tparam T The coordinates type. * @param unsorted_fragment_cell_ranges The unsorted fragment cell ranges. * @param fragment_cell_ranges The sorted fragment cell ranges output by * the function as a result. * @return TILEDB_ARS_OK on success and TILEDB_ARS_ERR on error. */ template int sort_fragment_cell_ranges( std::vector& unsorted_fragment_cell_ranges, FragmentCellRanges& fragment_cell_ranges) const; }; /** * Class of fragment cell range objects used in the priority queue algorithm. */ template class ArrayReadState::PQFragmentCellRange { public: /** * Constructor. * * @param array_schema The schema of the array. * @param fragment_read_states The read states of all fragments in the array. */ PQFragmentCellRange( const ArraySchema* array_schema, const std::vector* fragment_read_states); /** Returns true if the fragment the range belongs to is dense. */ bool dense() const; /** * Returns true if the calling object begins after the end of the input * range. */ bool begins_after(const PQFragmentCellRange* fcr) const; /** Returns true if the calling object ends after the input range. */ bool ends_after(const PQFragmentCellRange* fcr) const; /** Exports information to a fragment cell range. */ void export_to(FragmentCellRange& fragment_cell_range); /** Imports information from a fragment cell range. */ void import_from(const FragmentCellRange& fragment_cell_range); /** * Returns true if the calling object range must be split by the input * range. */ bool must_be_split(const PQFragmentCellRange* fcr) const; /** * Returns true if the input range must be trimmed by the callling object. */ bool must_trim(const PQFragmentCellRange* fcr) const; /** * Splits the calling object into two ranges based on the first input. The * first range will replace the calling object. The second range will be * stored in the second input. The third input is necessary for the * splitting. */ void split( const PQFragmentCellRange* fcr, PQFragmentCellRange* fcr_new, const T* tile_domain); /** * Splits the calling object into three ranges based on the input fcr. * - First range: Non-overlapping part of calling object range, stored * at fcr_left. * - Second range: A unary range at the left end point of the * first input, stored at fcr_unary. Note that this may not exist. * - Third range: The updated calling object range, which is trimmed to * start after the unary range. */ void split_to_3( const PQFragmentCellRange* fcr, PQFragmentCellRange* fcr_left, PQFragmentCellRange* fcr_unary); /** * Trims the first input range to the non-overlapping range stored in * the second input range. If the cell range of fcr_trimmed is NULL, * then fcr_trimmed is empty. The third input is necessary for the * trimming. */ void trim( const PQFragmentCellRange* fcr, PQFragmentCellRange* fcr_trimmed, const T* tile_domain) const; /** Returns true if the range is unary. */ bool unary() const; /** The cell range as a pair of coordinates. */ T* cell_range_; /** The fragment id. */ int fragment_id_; /** The tile id of the left endpoint of the cell range. */ int64_t tile_id_l_; /** The tile id of the right endpoint of the cell range. */ int64_t tile_id_r_; /** The position on disk of the tile corresponding to the cell range. */ int64_t tile_pos_; private: /** The array schema. */ const ArraySchema* array_schema_; /** Size of coordinates. */ size_t coords_size_; /** Dimension number. */ int dim_num_; /** Stores the read state of each fragment in the array. */ const std::vector* fragment_read_states_; }; /** * Wrapper of comparison function in the priority queue of the fragment cell * ranges. */ template class ArrayReadState::SmallerPQFragmentCellRange { public: /** Constructor. */ SmallerPQFragmentCellRange(); /** Constructor. */ SmallerPQFragmentCellRange(const ArraySchema* array_schema); /** * Comparison operator. First the smallest tile id of the left range end point * wins, then the smallest start range endpoint, then the largest fragment id. */ bool operator () ( PQFragmentCellRange* a, PQFragmentCellRange* b) const; private: /** The array schema. */ const ArraySchema* array_schema_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array_schema.h000066400000000000000000001132001453617025200246420ustar00rootroot00000000000000/** * @file array_schema.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ArraySchema. */ #ifndef __ARRAY_SCHEMA_H__ #define __ARRAY_SCHEMA_H__ #include "array_schema_c.h" #include "metadata_schema_c.h" #include "hilbert_curve.h" #include "storage_fs.h" #include #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Special key dimension name. */ #define TILEDB_AS_KEY_DIM1_NAME "__key_dim_1" #define TILEDB_AS_KEY_DIM2_NAME "__key_dim_2" #define TILEDB_AS_KEY_DIM3_NAME "__key_dim_3" #define TILEDB_AS_KEY_DIM4_NAME "__key_dim_4" /**@}*/ /**@{*/ /** Return code. */ #define TILEDB_AS_OK 0 #define TILEDB_AS_ERR -1 /**@}*/ /** Default parameters. */ #define TILEDB_AS_CAPACITY 10000 /** Default error message. */ #define TILEDB_AS_ERRMSG std::string("[TileDB::ArraySchema] Error: ") /** * The first 4 bytes of the array schema is the version. Bump this version whenever * the schema is changed **/ #define TILEDB_ARRAY_SCHEMA_VERSION_TAG 0x2u /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_as_errmsg; /** Compression fields are stored as 1 byte in the schema, the last 4 least significant digits * denote the main compression type, the next 2 denote pre compression filters and the leading 2 digits * denote the post compression filters. * Compression fields can be simply composed as a sum of 1 main compression type and optionally one pre * and one post compression type. * e.g. compression for an attribute could be TILEDB_GZIP+TILEDB_DELTA_ENCODE+TILEDB_CHECKSUM * ^^^^pre^^^^ ^^^^post^^^^ */ typedef enum filter_type_t {COMPRESS=0xF, PRE_COMPRESS=0x30, POST_COMPRESS=0xC0} filter_type_t; /** Specifies the array schema. */ class ArraySchema { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ ArraySchema(StorageFS *fs); /** Destructor. */ ~ArraySchema(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns the array workspace. */ const std::string& array_workspace() const; /** Returns the array name. */ const std::string& array_name() const; /** Exports the array schema into the input array schema struct. */ void array_schema_export(ArraySchemaC* array_schema_c) const; /** Exports the array schema into the input metadata schema struct. */ void array_schema_export(MetadataSchemaC* metadata_schema_c) const; /** Returns the name of the attribute with the input id. */ const std::string& attribute(int attribute_id) const; /** * Returns the id of the input attribute. * * @param attribute The attribute name whose id will be retrieved. * @return The attribute id if the input attribute exists, and TILEDB_AS_ERR * otherwise. */ int attribute_id(const std::string& attribute) const; /** Returns the number of attributes. */ int attribute_num() const; /** Returns the attributes. */ const std::vector& attributes() const; /** Returns the capacity. */ int64_t capacity() const; /** * Returns the number of cells per tile. Meaningful only for the dense case. */ int64_t cell_num_per_tile() const; /** Returns the cell order. */ int cell_order() const; /** Returns the size of cell on the input attribute. */ size_t cell_size(int attribute_id) const; /** Returns the number of values per cell of the input attribute. */ int cell_val_num(int attribute_id) const; /** Returns the compression type of the input attribute. */ int compression(int attribute_id) const; /** Returns the compression level of the input attribute */ int compression_level(int attribute_id) const; /** * Returns the compression type associated with the offsets to the input attribute. * This is only relevant if the number of cells for the attribute is variable. */ int offsets_compression(int attribute_id) const; /** * Returns the compression level associated with the offsets to the input attribute. * This is only relevant if the number of cells for the attribute is variable. */ int offsets_compression_level(int attribute_id) const; /** Returns the coordinates size. */ size_t coords_size() const; /** Returns the type of the coordinates. */ int coords_type() const; /** True if the array is dense. */ bool dense() const; /** Returns the number of dimensions. */ int dim_num() const; /** Returns the domain. */ const void* domain() const; /** * Gets the ids of the input attributes. * * @param attributes The name of the attributes whose ids will be retrieved. * @param attribute_ids The ids that are retrieved by the function. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int get_attribute_ids( const std::vector& attributes, std::vector& attribute_ids) const; /** * Returns true if the input range is contained fully in a single * column of tiles. */ bool is_contained_in_tile_slab_col(const void* range) const; /** * Returns true if the input range is contained fully in a single * column of tiles. * * @tparam T The coordinates type. * @param range The input range. * @return True if the input range is contained fully in a single * column of tiles. */ template bool is_contained_in_tile_slab_col(const T* range) const; /** * Returns true if the input range is contained fully in a single * row of tiles. */ bool is_contained_in_tile_slab_row(const void* range) const; /** * Returns true if the input range is contained fully in a single * row of tiles. * * @tparam T The coordinates type. * @param range The input range. * @return True if the input range is contained fully in a single * row of tiles. */ template bool is_contained_in_tile_slab_row(const T* range) const; /** Prints information about the array schema to stdout. */ void print() const; /** * Serializes the array schema object into a binary buffer. * * @param array_schema_bin The binary buffer to be created and populated * by the function with the object data. Note that the caller is * responsible for releasing this buffer afterwards. * @param array_schema_bin_size The size of the created binary buffer. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int serialize(void*& array_schema_bin, size_t& array_schema_bin_size) const; /** * Returns the type of overlap of the input subarrays. * * @tparam T The types of the subarrays. * @param subarray_a The first input subarray. * @param subarray_b The second input subarray. * @param overlap_subarray The overlap area between *subarray_a* and * *subarray_b*. * @return The type of overlap, which can be one of the following: * - 0: No overlap * - 1: *subarray_a* fully covers *subarray_b* * - 2: Partial overlap (non-contig) * - 3: Partial overlap (contig) */ template int subarray_overlap( const T* subarray_a, const T* subarray_b, T* overlap_subarray) const; /** Returns the tile domain. */ const void* tile_domain() const; /** Returns the tile extents. */ const void* tile_extents() const; /** * Returns the number of tiles in the array domain (applicable only to dense * arrays). */ int64_t tile_num() const; /** * Returns the number of tiles in the array domain (applicable only to dense * arrays). * * @tparam T The coordinates type. * @return The number of tiles. */ template int64_t tile_num() const; /** * Returns the number of tiles overlapping with the input range * (applicable only to dense arrays). */ int64_t tile_num(const void* range) const; /** * Returns the number of tiles in the input domain (applicable only to dense * arrays). * * @tparam T The coordinates type. * @param domain The input domain. * @return The number of tiles. */ template int64_t tile_num(const T* domain) const; /** Returns the tile order. */ int tile_order() const; /** Return the number of cells in a column tile slab of an input subarray. */ int64_t tile_slab_col_cell_num(const void* subarray) const; /** Return the number of cells in a row tile slab of an input subarray. */ int64_t tile_slab_row_cell_num(const void* subarray) const; /** Returns the type of the i-th attribute, or NULL if 'i' is invalid. */ int type(int i) const; /** Returns the type size of the i-th attribute. */ size_t type_size(int i) const; /** Returns the number of attributes with variable-sized values. */ int var_attribute_num() const; /** Returns *true* if the indicated attribute has variable-sized values. */ bool var_size(int attribute_id) const; /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** * It assigns values to the members of the object from the input buffer. * * @param array_schema_bin The input binary buffer with the object data. * @param array_schema_bin_size The size of the input binary buffer. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int deserialize(const void* array_schema_bin, size_t array_schema_bin_size); /** * Initializes the ArraySchema object using the information provided in the * input C-style ArraySchemaC struct. * * @param array_schema_c The array schema in a C-style struct. * @param do_print Option to print array_schema * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int init(const ArraySchemaC* array_schema_c, bool do_print=false); /** * Initializes the ArraySchema object using the information provided in the * inpur C-style MetadataSchemaC struct. * * @param metadata_schema_c The metadata schema in a C-style struct. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int init(const MetadataSchemaC* metadata_schema_c); /** Sets the array workspace. */ void set_array_workspace(const char* array_workspace); /** Sets the array name. */ void set_array_name(const char* array_name); /** * Sets attribute names. There should not be any duplicate names. Moreover, * there should not be an attribute with the same name as a dimension. * * @param attributes The attribute names. * @param attribute_num The number of attributes. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int set_attributes(char** attributes, int attribute_num); /** Sets the tile capacity. */ void set_capacity(int64_t capacity); /** Sets the number of cell values per attribute. */ void set_cell_val_num(const int* cell_val_num); /** * Sets the cell order. Supported cell orders: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR * - TILEDB_HILBSERT * * @param cell_order The cell order. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int set_cell_order(int cell_order); /** Sets the compression types. */ int set_compression(int* compression); /** Sets the compression levels. */ int set_compression_level(int* compression_level); /** Sets the offsets compression types. */ int set_offsets_compression(int* compression); /** Sets the offsets compression levels. */ int set_offsets_compression_level(int* compression_level); /** Sets the proper flag to indicate if the array is dense. */ void set_dense(int dense); /** * Sets dimension names. There should not be any duplicate names. Moreover, * there should not be a dimension with the same name as an attribute. * * @param dimensions The dimension names. * @param dim_num The number of dimensions. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int set_dimensions(char** dimensions, int dim_num); /** * Sets the domain. * * @param domain The domain. It should contain one [lower, upper] pair per * dimension. Thie type of the values stored in this buffer should match * the coordinates type. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. * * @note The dimensions and types must already have been set before calling * this function. */ int set_domain(const void* domain); /** * Sets the tile extents. * * @param tile_extents The tile extents (only applicable to regular tiles). * There should be one value for each dimension. The type of the values * stored in this buffer should match the coordinates type. If it is NULL, * then it means that the array has irregular tiles (and, hence, it is * sparse). * * @note The dimensions and types must already have been set prior to calling * this function. Moreover, dense arrays must always have tile extents, * whereas arrays defined as key-value stores must not have tile extents. */ int set_tile_extents(const void* tile_extents); /** * Sets the tile order. Supported tile orders. * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR * - TILEDB_HILBSERT * * @param tile_order The tile order. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. */ int set_tile_order(int tile_order); /** * Sets the types. There should be one type per attribute plus one (the last * one) for the coordinates. * The supported types for the attributes are: * - TILEDB_CHAR, TILEDB_INT8, TILEDB_UINT8 * - TILEDB_INT16, TILEDB_UINT16 * - TILEDB_INT32, TILEDB_UINT32 * - TILEDB_INT64, TILDB_UINT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * * The supported types for the coordinates are: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * * @param types The types. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. * * @note The attributes, dimensions and the dense flag must have already been * set before calling this function. */ int set_types(const int* types); /* ********************************* */ /* MISC */ /* ********************************* */ /** * Checks the cell order of the input coordinates. Note that, in the presence * of a regular tile grid, this function assumes that the cells are in the * same regular tile. * * @tparam T The coordinates type. * @param coords_a The first input coordinates. * @param coords_b The second input coordinates. * @return One of the following: * - -1 if the first coordinates precede the second * - 0 if the two coordinates are identical * - +1 if the first coordinates succeed the second */ template int cell_order_cmp(const T* coords_a, const T* coords_b) const; /** * Expands the input domain such that it coincides with the boundaries of * the array's regular tiles (i.e., it maps it on the regular tile grid). * If the array has no regular tile grid, the function does not do anything. * * @param domain The domain to be expanded. * @return void */ void expand_domain(void* domain) const; /** * Expands the input domain such that it coincides with the boundaries of * the array's regular tiles (i.e., it maps it on the regular tile grid). * If the array has no regular tile grid, the function does not do anything. * * @tparam The domain type. * @param domain The domain to be expanded. * @return void */ template void expand_domain(T* domain) const; /** * Returns the position of the input coordinates inside its corresponding * tile, based on the array cell order. Applicable only to **dense** arrays. * * @tparam T The coordinates type. * @param coords The input coordindates, which are expressed as global * coordinates in the array domain. * @return The position of the cell coordinates in the array cell order * within its corresponding tile. In case of error, the function returns * TILEDB_AS_ERR. */ template int64_t get_cell_pos(const T* coords) const; /** * Retrieves the next coordinates along the array cell order within a given * domain (desregarding whether the domain is split into tiles or not). * Applicable only to **dense** arrays. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param cell_coords The input cell coordinates, which the function modifies * to store the next coordinates at termination. * @param coords_retrieved Will store true if the retrieved coordinates are * inside the domain, and false otherwise. * @return void */ template void get_next_cell_coords( const T* domain, T* cell_coords, bool& coords_retrieved) const; /** * Retrieves the next tile coordinates along the array tile order within a * given tile domain. Applicable only to **dense** arrays. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param tile_coords The input tile coordinates, which the function modifies * to store the next tile coordinates at termination. * @return void */ template void get_next_tile_coords(const T* domain, T* tile_coords) const; /** * Retrieves the previous coordinates along the array cell order within a * given domain (desregarding whether the domain is split into tiles or not). * Applicable only to **dense** arrays. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param cell_coords The input cell coordinates, which the function modifies * to store the previous coordinates at termination. * @return void */ template void get_previous_cell_coords(const T* domain, T* cell_coords) const; /** * Gets a subarray of tile coordinates for the input (cell) subarray * over the input array domain. Retrieves also the tile domain of * the array.. * * @tparam T The domain type. * @param subarray The input (cell) subarray. * @param tile_domain The array tile domain to be retrieved. * @param subarray_in_tile_domain The output (tile) subarray. * @return void */ template void get_subarray_tile_domain( const T* subarray, T* tile_domain, T* subarray_in_tile_domain) const; /** * Returns the tile position along the array tile order within the input * domain. Applicable only to **dense** arrays. * * @tparam T The domain type. * @param tile_coords The tile coordinates. * @return The tile position of *tile_coords* along the tile order of the * array inside the array domain, or TILEDB_AS_ERR on error. */ template int64_t get_tile_pos(const T* tile_coords) const; /** * Returns the tile position along the array tile order within the input * domain. Applicable only to **dense** arrays. * * @tparam T The domain type. * @param domain The input domain, which is a cell domain partitioned into * regular tiles in the same manner as that of the array domain (however * *domain* may be a sub-domain of the array domain). * @param tile_coords The tile coordinates. * @return The tile position of *tile_coords* along the tile order of the * array inside the input domain, or TILEDB_AS_ERR on error. */ template int64_t get_tile_pos( const T* domain, const T* tile_coords) const; /** * Gets the tile subarray for the input tile coordinates. * * @tparam T The coordinates type. * @param tile_coords The input tile coordinates. * @param tile_subarray The output tile subarray. * @return void. */ template void get_tile_subarray(const T* tile_coords, T* tile_subarray) const; /** * Returns the Hilbert id of the input coordinates. * * @tparam T The coordinates type. * @param coords The coordinates for which the Hilbert id is computed. * @return The computed Hilbert id. */ template int64_t hilbert_id(const T* coords) const; /** * Checks the order of the input coordinates. First the tile order is checked * (which, in case of non-regular tiles, is always the same), breaking the * tie by checking the cell order. * * @tparam T The coordinates type. * @param coords_a The first input coordinates. * @param coords_b The second input coordinates. * @return One of the following: * - -1 if the first coordinates precede the second * - 0 if the two coordinates are identical * - +1 if the first coordinates succeed the second */ template int tile_cell_order_cmp(const T* coords_a, const T* coords_b) const; /** * Returns the id of the tile the input coordinates fall into. * * @tparam T The coordinates type. * @param cell_coords The input coordinates. * @return The computed tile id. */ template int64_t tile_id(const T* cell_coords) const; /** * Checks the tile order of the input coordinates. * * @tparam T The coordinates type. * @param coords_a The first input coordinates. * @param coords_b The second input coordinates. * @return One of the following: * - -1 if the first coordinates precede the second on the tile order * - 0 if the two coordinates have the same tile order * - +1 if the first coordinates succeed the second on the tile order */ template int tile_order_cmp(const T* coords_a, const T* coords_b) const; /** * Returns true if the schema has a version tag */ bool version_tag_exists() const; /** * Get version for the schema */ uint32_t get_version() const; /* ********************************* */ /* AUXILIARY ATTRIBUTES */ /* ********************************* */ /** * Auxiliary attribute used in the computation of tile ids, in order to avoid * repeated allocations and deallocations that impact performance. */ void* tile_coords_aux_; private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** * The array workspace directory. */ std::string array_workspace_; /** * The array name. It is a directory, whose parent must be a TileDB workspace, * or group. */ std::string array_name_; /** The attribute names. */ std::vector attributes_; /** The number of attributes. */ int attribute_num_; /** * The tile capacity for the case of sparse fragments. */ int64_t capacity_; /** The number of cells per tile. Meaningful only for the **dense** case. */ int64_t cell_num_per_tile_; /** * The cell order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR * - TILEDB_HILBERT. */ int cell_order_; /** Stores the size of every attribute (plus coordinates in the end). */ std::vector cell_sizes_; /** * Specifies the number of values per attribute for a cell. If it is NULL, * then each attribute has a single value per cell. If for some attribute * the number of values is variable (e.g., in the case off strings), then * TILEDB_VAR_NUM must be used. */ std::vector cell_val_num_; /** * The compression type for each attribute (plus one extra at the end for the * coordinates. It can be one of the following: * - TILEDB_NO_COMPRESSION * - TILEDB_GZIP * - TILEDB_ZSTD * - TILEDB_LZ4 * - TILEDB_BLOSC * - TILEDB_BLOSC_LZ4 * - TILEDB_BLOSC_LZ4HC * - TILEDB_BLOSC_SNAPPY * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE */ std::vector compression_; /** * The compression level for each attribute + 1 for coordinates. This level will be interpreted * based on the compression type in compression_. Introduced in schema version "1". */ std::vector compression_level_; /** * The compression type for the offsets associated with the attribute. It is only relevant for * attributes that have variable number of cells(TILEDB_VAR_NUM). Introduced in schema version "2". */ std::vector offsets_compression_; /** * The compression level for the offsets associated with the attribute. It is only relevant for * attributes that have variable number of cells(TILEDB_VAR_NUM). Introduced in schema version "2". */ std::vector offsets_compression_level_; /** Auxiliary variable used when calculating Hilbert ids. */ int* coords_for_hilbert_; /** The size (in bytes) of the coordinates. */ size_t coords_size_; /** * Specifies if the array is dense or sparse. If the array is dense, * then the user must specify tile extents (see below). */ bool dense_; /** The dimension names. */ std::vector dimensions_; /** The number of dimensions. */ int dim_num_; /** * The array domain. It should contain one [lower, upper] pair per dimension. * The type of the values stored in this buffer should match the coordinates * type. */ void* domain_; /** * Number of bits used for the calculation of cell ids with the * Hilbert curve. */ int hilbert_bits_; /** A Hilbert curve object for finding cell ids. */ HilbertCurve* hilbert_curve_; /** * The array domain. It should contain one [lower, upper] pair per dimension. * The type of the values stored in this buffer should match the coordinates * type. */ void* tile_domain_; /** * The tile extents (only applicable to regular tiles). There should be one * value for each dimension. The type of the values stored in this buffer * should match the coordinates type. If it is NULL, then it means that the * array has irregular tiles (and, hence, it is sparse). */ void* tile_extents_; /** * Offsets for calculating tile positions and ids for the column-major * tile order. */ std::vector tile_offsets_col_; /** * Offsets for calculating tile positions and ids for the row-major * tile order. */ std::vector tile_offsets_row_; /** * The tile order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR. */ int tile_order_; /** * The attribute types, plus an extra one in the end for the coordinates. * The attribute type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR * * The coordinate type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 */ std::vector types_; /** Stores the size of every attribute type (plus coordinates in the end). */ std::vector type_sizes_; /** Array schema version **/ uint32_t version_tag_; /** The Storage Filesystem */ StorageFS *fs_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Computes and returns the size of the binary representation of the * ArraySchema object. */ size_t compute_bin_size() const; /** * Compute the number of cells per tile. Meaningful only for the **dense** * case. * * @return void */ void compute_cell_num_per_tile(); /** * Compute the number of cells per tile. Meaningful only for the **dense** * case. * * @tparam T The coordinates type. * @return void */ template void compute_cell_num_per_tile(); /** Computes and returns the size of an attribute (or coordinates). */ size_t compute_cell_size(int attribute_id) const; /** * Computes the number of bits per dimension required by the Hilbert curve. * * @tparam T The domain type. * @return void */ template void compute_hilbert_bits(); /** * Computes the tile domain. Applicable only to arrays with regular tiles. * * @return void */ void compute_tile_domain(); /** * Computes tile offsets neccessary when computing tile positions and ids. * * @return void */ void compute_tile_offsets(); /** * Computes tile offsets neccessary when computing tile positions and ids. * * @tparam T The coordinates type. * @return void */ template void compute_tile_offsets(); /** * Computes the tile domain. Applicable only to arrays with regular tiles. * * @tparam T The domain type. * @return void */ template void compute_tile_domain(); /** Computes and returns the size of a type. */ size_t compute_type_size(int attribute_id) const; /** * Returns the position of the input coordinates inside its corresponding * tile, based on the array cell order. Applicable only to **dense** arrays, * and focusing on the **column-major** cell order. * * @tparam T The coordinates type. * @param coords The input coordindates, which are expressed as global * coordinates in the array domain. * @return The position of the cell coordinates in the array cell order * within its corresponding tile. In case of error, the function returns * TILEDB_AS_ERR. */ template int64_t get_cell_pos_col(const T* coords) const; /** * Returns the position of the input coordinates inside its corresponding * tile, based on the array cell order. Applicable only to **dense** arrays, * and focusing on the **row-major** cell order. * * @tparam T The coordinates type. * @param coords The input coordindates, which are expressed as global * coordinates in the array domain. * @return The position of the cell coordinates in the array cell order * within its corresponding tile. In case of error, the function returns * TILEDB_AS_ERR. */ template int64_t get_cell_pos_row(const T* coords) const; /** * Retrieves the next coordinates along the array cell order within a given * domain (desregarding whether the domain is split into tiles or not). * Applicable only to **dense** arrays, and focusing on **column-major** * cell order. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param cell_coords The input cell coordinates, which the function modifies * to store the next coordinates at termination. * @param coords_retrieved Will store true if the retrieved coordinates are * inside the domain, and false otherwise. * @return void */ template void get_next_cell_coords_col( const T* domain, T* cell_coords, bool& coords_retrieved) const; /** * Retrieves the next coordinates along the array cell order within a given * domain (desregarding whether the domain is split into tiles or not). * Applicable only to **dense** arrays, and focusing on **row-major** * cell order. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param cell_coords The input cell coordinates, which the function modifies * to store the next coordinates at termination. * @param coords_retrieved Will store true if the retrieved coordinates are * inside the domain, and false otherwise. * @return void */ template void get_next_cell_coords_row( const T* domain, T* cell_coords, bool& coords_retrieved) const; /** * Retrieves the next tile coordinates along the array tile order within a * given tile domain. Applicable only to **dense** arrays, and focusing on * the **column-major** tile order. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param tile_coords The input tile coordinates, which the function modifies * to store the next tile coordinates at termination. * @return void */ template void get_next_tile_coords_col(const T* domain, T* tile_coords) const; /** * Retrieves the next tile coordinates along the array tile order within a * given tile domain. Applicable only to **dense** arrays, and focusing on * the **row-major** tile order. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param tile_coords The input tile coordinates, which the function modifies * to store the next tile coordinates at termination. * @return void */ template void get_next_tile_coords_row(const T* domain, T* tile_coords) const; /** * Retrieves the previous coordinates along the array cell order within a * given domain (desregarding whether the domain is split into tiles or not). * Applicable only to **dense** arrays, and focusing on the **column-major** * cell order. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param cell_coords The input cell coordinates, which the function modifies * to store the previous coordinates at termination. * @return void */ template void get_previous_cell_coords_col(const T* domain, T* cell_coords) const; /** * Retrieves the previous coordinates along the array cell order within a * given domain (desregarding whether the domain is split into tiles or not). * Applicable only to **dense** arrays, and focusing on the **row-major** * cell order. * * @tparam T The coordinates type. * @param domain The targeted domain. * @param cell_coords The input cell coordinates, which the function modifies * to store the previous coordinates at termination. * @return void */ template void get_previous_cell_coords_row(const T* domain, T* cell_coords) const; /** * Returns the tile position along the array tile order within the input * domain. Applicable only to **dense** arrays, and focusing on the * **column-major** tile order. * * @tparam T The domain type. * @param tile_coords The tile coordinates. * @return The tile position of *tile_coords* along the tile order of the * array inside the array domain. */ template int64_t get_tile_pos_col(const T* tile_coords) const; /** * Returns the tile position along the array tile order within the input * domain. Applicable only to **dense** arrays, and focusing on the * **column-major** tile order. * * @tparam T The domain type. * @param domain The input domain, which is a cell domain partitioned into * regular tiles in the same manner as that of the array domain * (however *domain* may be a sub-domain of the array domain). * @param tile_coords The tile coordinates. * @return The tile position of *tile_coords* along the tile order of the * array inside the input domain. */ template int64_t get_tile_pos_col( const T* domain, const T* tile_coords) const; /** * Returns the tile position along the array tile order within the input * domain. Applicable only to **dense** arrays, and focusing on the * **row-major** tile order. * * @tparam T The domain type. * @param tile_coords The tile coordinates. * @return The tile position of *tile_coords* along the tile order of the * array inside the array domain. */ template int64_t get_tile_pos_row(const T* tile_coords) const; /** * Returns the tile position along the array tile order within the input * domain. Applicable only to **dense** arrays, and focusing on the * **row-major** tile order. * * @tparam T The domain type. * @param domain The input domain, which is a cell domain partitioned into * regular tiles in the same manner as that of the array domain (however * *domain* may be a sub-domain of the array domain). * @param tile_coords The tile coordinates. * @return The tile position of *tile_coords* along the tile order of the * array inside the input domain. */ template int64_t get_tile_pos_row( const T* domain, const T* tile_coords) const; /** Initializes a Hilbert curve. */ void init_hilbert_curve(); /** Return the number of cells in a column tile slab of an input subarray. */ template int64_t tile_slab_col_cell_num(const T* subarray) const; /** Return the number of cells in a row tile slab of an input subarray. */ template int64_t tile_slab_row_cell_num(const T* subarray) const; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array_schema_c.h000066400000000000000000000113701453617025200251510ustar00rootroot00000000000000/** * @file array_schema_c.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * A C-style struct that specifies the array schema. */ #ifndef __ARRAY_SCHEMA_C_H__ #define __ARRAY_SCHEMA_C_H__ #include /** Specifies the array schema. */ typedef struct ArraySchemaC { /** * The array workspace directory. */ char* array_workspace_; /** * The array name. It is a directory, whose parent must be a TileDB workspace, * or group. */ char* array_name_; /** The attribute names. */ char** attributes_; /** The number of attributes. */ int attribute_num_; /** * The tile capacity for the case of sparse fragments. If it is <=0, * TileDB will use its default. */ int64_t capacity_; /** * The cell order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR * - TILEDB_HILBERT. */ int cell_order_; /** * Specifies the number of values per attribute for a cell. If it is NULL, * then each attribute has a single value per cell. If for some attribute * the number of values is variable (e.g., in the case off strings), then * TILEDB_VAR_NUM must be used. */ int* cell_val_num_; /** * The compression type for each attribute (plus one extra at the end for the * coordinates. It can be one of the following: * - TILEDB_NO_COMPRESSION * - TILEDB_GZIP * - TILEDB_ZSTD * - TILEDB_LZ4 * - TILEDB_BLOSC * - TILEDB_BLOSC_LZ4 * - TILEDB_BLOSC_LZ4HC * - TILEDB_BLOSC_SNAPPY * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE */ int* compression_; /** The level of compression for each attribute */ int* compression_level_; /** * The compression type for the offsets associated with the attribute. It is only relevant for * attributes that have variable number of cells(TILEDB_VAR_NUM). Introduced in schema version "2". */ int* offsets_compression_; /** * The compression level for the offsets associated with the attribute. It is only relevant for * attributes that have variable number of cells(TILEDB_VAR_NUM). Introduced in schema version "2". */ int* offsets_compression_level_; /** * Specifies if the array is dense (1) or sparse (0). If the array is dense, * then the user must specify tile extents (see below). */ int dense_; /** The dimension names. */ char** dimensions_; /** The number of dimensions. */ int dim_num_; /** * The array domain. It should contain one [lower, upper] pair per dimension. * The type of the values stored in this buffer should match the coordinates * type. */ void* domain_; /** * The tile extents. There should be one value for each dimension. The type of * the values stored in this buffer should match the coordinates type. If it * is NULL (applicable only to sparse arrays), then it means that the * array has irregular tiles. */ void* tile_extents_; /** * The tile order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR. */ int tile_order_; /** * The attribute types, plus an extra one in the end for the coordinates. * The attribute type can be one of the following: * - TILEDB_INT32, TILEDB_UINT32 * - TILEDB_INT64, TILEDB_UINT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR, TILEDB_INT8, TILEDB_UINT8 * - TILEDB_INT16, TILEDB_UINT16 * * The coordinate type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 */ int* types_; } ArraySchemaC; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array_sorted_read_state.h000066400000000000000000000745531453617025200271160ustar00rootroot00000000000000/** * @file array_sorted_read_state.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ArraySortedReadState. */ #ifndef __ARRAY_SORTED_READ_STATE_H__ #define __ARRAY_SORTED_READ_STATE_H__ #include "array.h" #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_ASRS_OK 0 #define TILEDB_ASRS_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_ASRS_ERRMSG std::string("[TileDB::ArraySortedReadState] Error: ") /** Initial internal buffer size for the case of sparse arrays. */ #define TILEDB_ASRS_INIT_BUFFER_SIZE 10000000 // ~ 10MB /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_asrs_errmsg; class Array; /** * Stores the state necessary when reading cells from the array fragments, * sorted in a way different to the global cell order. */ class ArraySortedReadState { public: /* ********************************* */ /* TYPE DEFINITIONS */ /* ********************************* */ /** Used in functors. */ struct ASRS_Data { /** An id (typically an attribute id or a tile slab id. */ int id_; /** Another id (typically a tile id). */ int64_t id_2_; /** The calling object. */ ArraySortedReadState* asrs_; }; /** Stores state about the current read/copy request. */ struct CopyState { /** Current offsets in user buffers. */ size_t* buffer_offsets_; /** User buffer sizes. */ size_t* buffer_sizes_; /** User buffers. */ void** buffers_; }; /** Info about a tile slab. */ struct TileSlabInfo { /** Used in calculations of cell ids, one vector per tile. */ int64_t** cell_offset_per_dim_; /** Cell slab size per attribute per tile. */ size_t** cell_slab_size_; /** Number of cells in a cell slab per tile. */ int64_t* cell_slab_num_; /** * The range overlap of the **normalized** tile slab with each * **normalized** tile range. */ void** range_overlap_; /** * Start offsets of each tile in the local buffer, per attribute per tile. */ size_t** start_offsets_; /** Number of tiles in the tile slab. */ int64_t tile_num_; /** Used in calculations of tile ids. */ int64_t* tile_offset_per_dim_; }; /** The state for a tile slab copy. */ struct TileSlabState { /** Keeps track of whether a tile slab copy for an attribute id done. */ bool* copy_tile_slab_done_; /** * Applicable only to the sparse case. It holds the current cell position * to be considered, per attribute. */ int64_t* current_cell_pos_; /** Current coordinates in tile slab per attribute. */ void** current_coords_; /** * The offset in the local buffers of the next cell slab to be copied per * attribute. Note that this applies only to fixed-sized attributes * because the offsets of the variable-sized attributes can be derived from * the buffers that hold the fixed-sized offsets. */ size_t* current_offsets_; /** The current tile per attribute. */ int64_t* current_tile_; }; /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param array The array this array sorted read state belongs to. */ ArraySortedReadState(Array* array); /** Destructor. */ ~ArraySortedReadState(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns true if the current slab is finished being copied. */ bool copy_tile_slab_done() const; /** True if read is done for all attributes. */ bool done() const; /** Returns true if copying into the user buffers resulted in overflow. */ bool overflow() const; /** * Returns true if copying into the user buffers resulted in overflow, for * the input attribute id. */ bool overflow(int attribute_id) const; /** * Same as Array::read(), but it sorts the cells in the buffers based on the * order the user specified in Array::init(). Note that this function will * fail if there is not enough system memory to hold the cells of a * 'tile slab' overlapping with the selected subarray. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * Array::init() or Array::reset_attributes(). The case of variable-sized * attributes is special. Instead of providing a single buffer for such * an attribute, **two** must be provided: the second will hold the * variable-sized cell values, whereas the first holds the start offsets * of each cell in the second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). The function will * attempt to write as many results as can fit in the buffers, and * potentially alter the buffer size to indicate the size of the *useful* * data written in the buffer. If a buffer cannot hold all results, the * function will still succeed, writing as much data as it can and turning * on an overflow flag which can be checked with function overflow(). The * next invocation will resume for the point the previous one stopped, * without inflicting a considerable performance penalty due to overflow. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int read(void** buffers, size_t* buffer_sizes); /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Initializes the array sorted read state. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int init(); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** Function for advancing a cell slab during a copy operation. */ void *(*advance_cell_slab_) (void*); /** AIO counter. */ int aio_cnt_; /** The AIO mutex conditions (one for each buffer). */ pthread_cond_t aio_cond_[2]; /** Data for the AIO requests. */ ASRS_Data aio_data_[2]; /** The current id of the buffers the next AIO will occur into. */ int aio_id_; /** The AIO mutex. */ pthread_mutex_t aio_mtx_; /** Indicates overflow per tile slab per attribute upon an AIO operation. */ bool* aio_overflow_[2]; /** AIO requests. */ AIO_Request aio_request_[2]; /** The status of the AIO requests.*/ int aio_status_[2]; /** The array this sorted read state belongs to. */ Array* array_; /** The ids of the attributes the array was initialized with. */ std::vector attribute_ids_; /** * The sizes of the attributes. For variable-length attributes, sizeof(size_t) * is stored. */ std::vector attribute_sizes_; /** Number of allocated buffers. */ int buffer_num_; /** Allocated sizes for buffers_ (similar to those used in Array::read). */ size_t* buffer_sizes_[2]; /** Temporary buffer sizes used in AIO requests. */ size_t* buffer_sizes_tmp_[2]; /** * Backup of temporary buffer sizes used in AIO requests (used when there is * overflow). */ size_t* buffer_sizes_tmp_bak_[2]; /** Local buffers (similar to those used in Array::read). */ void** buffers_[2]; /** Function for calculating cell slab info during a copy operation. */ void *(*calculate_cell_slab_info_) (void*); /** Function for calculating tile slab info during a copy operation. */ void *(*calculate_tile_slab_info_) (void*); /** * Used only in the sparse case. Holds the sorted positions of the cells * for the current tile slab to be copied. */ std::vector cell_pos_; /** * Used only in the sparse case. It is the element index in attribute_ids_ * that represents the coordinates attribute. */ int coords_attr_i_; /** * Used only in the sparse case. It is the element index in buffers_ * that represents the coordinates attribute. */ int coords_buf_i_; /** The coordinates size of the array. */ size_t coords_size_; /** The copy mutex conditions (one for each buffer). */ pthread_cond_t copy_cond_[2]; /** The current id of the buffers the next copy will occur from. */ int copy_id_; /** The copy state. */ CopyState copy_state_; /** The copy mutex. */ pthread_mutex_t copy_mtx_; /** The thread tha handles all the copying in the background. */ pthread_t copy_thread_; /** True if the copy thread is canceled. */ volatile bool copy_thread_canceled_; /** True if the copy thread is running. */ volatile bool copy_thread_running_; /** The number of dimensions in the array. */ int dim_num_; /** * Used only in the sparse case. It is true if the coordinates are not asked * by the user and, thus, TileDB had to append them as an extra attribute * to facilitate sorting the cell positions. * */ bool extra_coords_; /** The overflow mutex condition. */ pthread_cond_t overflow_cond_; /** The overflow mutex. */ pthread_mutex_t overflow_mtx_; /** Overflow flag for each attribute. */ bool* overflow_; /** * Overflow flag for each attribute. It starts with *true* for all * attributes, and becomes false once an attribute does not overflow any more. */ bool* overflow_still_; /** True if no more tile slabs to read. */ bool read_tile_slabs_done_; /** True if a copy must be resumed. */ bool resume_copy_; /** True if an AIO must be resumed. */ bool resume_aio_; /** The query subarray. */ void* subarray_; /** Auxiliary variable used in calculate_tile_slab_info(). */ void* tile_coords_; /** Auxiliary variable used in calculate_tile_slab_info(). */ void* tile_domain_; /** The tile slab to be read for the first and second buffers. */ void* tile_slab_[2]; /** Indicates if the tile slab has been initialized. */ bool tile_slab_init_[2]; /** Normalized tile slab. */ void* tile_slab_norm_[2]; /** The info for each of the two tile slabs under investigation. */ TileSlabInfo tile_slab_info_[2]; /** The state for the current tile slab being copied. */ TileSlabState tile_slab_state_; /** Wait for copy flags, one for each local buffer. */ bool wait_copy_[2]; /** Wait for AIO flags, one for each local buffer. */ bool wait_aio_[2]; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Advances a cell slab focusing on column-major order, and updates * the CopyState and TileSlabState. * Used in copy_tile_slab(). * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *advance_cell_slab_col_s(void* data); /** * Advances a cell slab focusing on row-major order, and updates * the CopyState and TileSlabState. * Used in copy_tile_slab(). * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *advance_cell_slab_row_s(void* data); /** * Advances a cell slab when the requested order is column-major. * * @tparam T The domain type. * @param aid The id of the attribute in attribute_ids_ to focus on. * @return void */ template void advance_cell_slab_col(int aid); /** * Advances a cell slab when the requested order is row-major. * * @tparam T The domain type. * @param aid The id of the attribute in attribute_ids_ to focus on. * @return void */ template void advance_cell_slab_row(int aid); /** * Called when an AIO completes. * * @param data A ASRS_Request object. * @return void */ static void *aio_done(void* data); /** True if some attribute overflowed for the input tile slab upon an AIO. */ bool aio_overflow(int aio_id); /** Sets the flag of wait_aio_[id] to true. */ void block_aio(int id); /** Sets the flag of wait_copy_[id] to true. */ void block_copy(int id); /** Sets the flag of resume_copy_ to true. */ void block_overflow(); /** * Calculate the attribute ids specified by the user upon array * initialization. */ void calculate_attribute_ids(); /** * Calculates the number of buffers to be allocated, based on the number * of attributes initialized for the array. * * @return void */ void calculate_buffer_num(); /** * Calculates the buffer sizes based on the array type. * * @return void */ void calculate_buffer_sizes(); /** * Calculates the buffer sizes based on the subarray and the number of cells * in a (full) tile slab. Applicable to dense arrays. * * @return void */ void calculate_buffer_sizes_dense(); /** * Calculates the buffer sizes based on configurable parameters. Applicable to * sparse arrays. * * @return void */ void calculate_buffer_sizes_sparse(); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *calculate_cell_slab_info_col_col_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *calculate_cell_slab_info_col_row_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order in row-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *calculate_cell_slab_info_row_col_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is row-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *calculate_cell_slab_info_row_row_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_col_col(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_col_row(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is row-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_row_row(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is row-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_row_col(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **array** cell order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_row(int id, int64_t tid); /** * Calculates the **normalized** tile domain overlapped by the input tile * slab. Note that this domain is the same for all tile slabs * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_domain(int id); /** * Calculates the info used in the copy_tile_slab() function. * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_slab_info(int id); /** * Calculates tile slab info for the case where the **array** tile order is * column-major * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *calculate_tile_slab_info_col(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **array** tile order is column-major. * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_slab_info_col(int id); /** * Calculates tile slab info for the case where the **array** tile order is * row-major * * @tparam T The domain type. * @param data Essentially a pointer to a ASRS_Data object. * @return void */ template static void *calculate_tile_slab_info_row(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **array** tile order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_slab_info_row(int id); /** * Function called by the copy thread. * * @param context This is practically the ArraySortedReadState object for * which the function is called (typically *this* is passed to this * argument by the caller). */ static void *copy_handler(void* context); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order. * Applicable to dense arrays. * * @return void. */ void copy_tile_slab_dense(); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order. * Applicable to sparse arrays. * * @return void. */ void copy_tile_slab_sparse(); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order, * focusing on a particular fixed-length attribute. * Applicable to dense arrays. * * @param aid The index on attribute_ids_ to focus on. * @param bid The index on the copy state buffers to focus on. * @return void. */ void copy_tile_slab_dense(int aid, int bid); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order, * focusing on a particular fixed-length attribute. * Applicable to sparse arrays. * * @param aid The index on attribute_ids_ to focus on. * @param bid The index on the copy state buffers to focus on. * @return void. */ void copy_tile_slab_sparse(int aid, int bid); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order, * focusing on a particular variable-length attribute. * Applicable to dense arrays. * * @param aid The index on attribute_ids_ to focus on. * @param bid The index on the copy state buffers to focus on. * @return void. */ void copy_tile_slab_dense_var(int aid, int bid); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order, * focusing on a particular variable-length attribute. * Applicable to sparse arrays. * * @param aid The index on attribute_ids_ to focus on. * @param bid The index on the copy state buffers to focus on. * @return void. */ void copy_tile_slab_sparse_var(int aid, int bid); /** * Creates the buffers based on the calculated buffer sizes. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int create_buffers(); /** Frees the copy state. */ void free_copy_state(); /** Frees the tile slab info. */ void free_tile_slab_info(); /** Frees the tile slab state. */ void free_tile_slab_state(); /** * Returns the cell id along the **array** order for the current coordinates * in the tile slab state for a particular attribute. * * @tparam T The domain type. * @param aid The targeted attribute. * @return The cell id. */ template int64_t get_cell_id(int aid); /** * Returns the tile id along the **array** order for the current coordinates * in the tile slab state for a particular attribute. * * @tparam T The domain type. * @param aid The targeted attribute. * @return The tile id. */ template int64_t get_tile_id(int aid); /** * Handles the copy requests. Applicable to dense arrays. * * @tparam T The domain type. * @return void. */ template void handle_copy_requests_dense(); /** * Handles the copy requests. Applicable to sparse arrays. * * @tparam T The domain type. * @return void. */ template void handle_copy_requests_sparse(); /** Initializes the AIO requests. */ void init_aio_requests(); /** Initializes the copy state. */ void init_copy_state(); /** Initializes the tile slab info. */ void init_tile_slab_info(); /** * Initializes the tile slab info for a particular tile slab, using the * input tile number. * * @tparam T The domain type. * @param id The slab id. * @return void. */ template void init_tile_slab_info(int id); /** Initializes the tile slab state. */ void init_tile_slab_state(); /** * Locks the AIO mutex. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int lock_aio_mtx(); /** * Locks the copy mutex. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int lock_copy_mtx(); /** * Locks the overflow mutex. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int lock_overflow_mtx(); /** * Retrieves the next column tile slab to be processed. Applicable to dense * arrays. * * @tparam T The domain type. * @return True if the next tile slab was retrieved, and false otherwise. */ template bool next_tile_slab_dense_col(); /** * Retrieves the next row tile slab to be processed. Applicable to dense * arrays. * * @tparam T The domain type. * @return True if the next tile slab was retrieved, and false otherwise. */ template bool next_tile_slab_dense_row(); /** * Retrieves the next column tile slab to be processed. Applicable to sparse * arrays. * * @tparam T The domain type. * @return True if the next tile slab was retrieved, and false otherwise. */ template bool next_tile_slab_sparse_col(); /** * Retrieves the next row tile slab to be processed. Applicable to sparse * arrays. * * @tparam T The domain type. * @return True if the next tile slab was retrieved, and false otherwise. */ template bool next_tile_slab_sparse_row(); /** * Same as Array::read(), but it sorts the cells in the buffers based on the * order the user specified in Array::init(). Note that this function will * fail if there is not enough system memory to hold the cells of a * 'tile slab' overlapping with the selected subarray. * * @tparam T The domain type. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ template int read(); /** * Same as read(), but the cells are placed in 'buffers' sorted in * column-major order with respect to the selected subarray. * Applicable only to dense arrays. * * @tparam T The domain type. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ template int read_dense_sorted_col(); /** * Same as read(), but the cells are placed in 'buffers' sorted in * row-major order with respect to the selected subarray. * Applicable only to dense arrays. * * @tparam T The domain type. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ template int read_dense_sorted_row(); /** * Same as read(), but the cells are placed in 'buffers' sorted in * column-major order with respect to the selected subarray. * Applicable only to sparse arrays. * * @tparam T The domain type. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ template int read_sparse_sorted_col(); /** * Same as read(), but the cells are placed in 'buffers' sorted in * row-major order with respect to the selected subarray. * Applicable only to sparse arrays. * * @tparam T The domain type. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ template int read_sparse_sorted_row(); /** * Reads the current tile slab into the input buffers. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int read_tile_slab(); /** * Signals an AIO condition. * * @param id The id of the AIO condition to be signaled. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int release_aio(int id); /** * Signals a copy condition. * * @param id The id of the copy condition to be signaled. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int release_copy(int id); /** * Signals the overflow condition. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int release_overflow(); /** Resets the AIO overflow flags for the input tile slab id. */ void reset_aio_overflow(int aio_id); /** Resets the temporary buffer sizes for the input tile slab id. */ void reset_buffer_sizes_tmp(int id); /** Resets the copy state using the input buffer info. */ void reset_copy_state(void** buffers, size_t* buffer_sizes); /** Resets the oveflow flags to **false**. */ void reset_overflow(); /** * Resets the tile_coords_ auxiliary variable. * * @tparam T The domain type. * @return void. */ template void reset_tile_coords(); /** * Resets the tile slab state. * * @tparam T The domain type. * @return void. */ template void reset_tile_slab_state(); /** * Sends an AIO request. * * @param aio_id The id of the tile slab the AIO request focuses on. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int send_aio_request(int aio_id); /** * It sorts the positions of the cells based on the coordinates * of the current tile slab to be copied. */ template void sort_cell_pos(); /** * Unlocks the AIO mutex. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int unlock_aio_mtx(); /** * Unlocks the copy mutex. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int unlock_copy_mtx(); /** * Unlocks the overflow mutex. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int unlock_overflow_mtx(); /** * Calculates the new tile and local buffer offset for the new (already * computed) current cell coordinates in the tile slab. * * @tparam T The domain type * @param aid The attribute id to focus on. * @return void. */ template void update_current_tile_and_offset(int aid); /** * Waits on a copy operation for the buffer with input id to finish. * * @param id The id of the buffer the copy operation must be completed. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int wait_copy(int id); /** * Waits on a AIO operation for the buffer with input id to finish. * * @param id The id of the buffer the AIO operation must be completed. * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int wait_aio(int id); /** * Waits until there is no buffer overflow. * * @return TILEDB_ASRS_OK for success and TILEDB_ASRS_ERR for error. */ int wait_overflow(); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/array/array_sorted_write_state.h000066400000000000000000000600741453617025200273260ustar00rootroot00000000000000/** * @file array_sorted_write_state.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ArraySortedWriteState. */ #ifndef __ARRAY_SORTED_WRITE_STATE_H__ #define __ARRAY_SORTED_WRITE_STATE_H__ #include "array.h" #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_ASWS_OK 0 #define TILEDB_ASWS_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_ASWS_ERRMSG \ std::string("[TileDB::ArraySortedWriteState] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_asws_errmsg; class Array; /** * It is responsibled for re-arranging the cells sorted in column- or row-major * order within the user subarray, such that they are sorted along the array * global cell order, and writes them into a new fragment. */ class ArraySortedWriteState { public: /* ********************************* */ /* TYPE DEFINITIONS */ /* ********************************* */ /** Used in functors. */ struct ASWS_Data { /** An id (typically an attribute id or a tile slab id. */ int id_; /** Another id (typically a tile id). */ int64_t id_2_; /** The calling object. */ ArraySortedWriteState* asws_; }; /** Stores local state about the current write/copy request. */ struct CopyState { /** Local buffer offsets. */ size_t* buffer_offsets_[2]; /** Local buffer sizes. */ size_t* buffer_sizes_[2]; /** Local buffers. */ void** buffers_[2]; }; /** Info about a tile slab. */ struct TileSlabInfo { /** Used in calculations of cell ids, one vector per tile. */ int64_t** cell_offset_per_dim_; /** Cell slab size per attribute per tile. */ size_t** cell_slab_size_; /** Number of cells in a cell slab per tile. */ int64_t* cell_slab_num_; /** * The range overlap of the **normalized** tile slab with each * **normalized** tile range. */ void** range_overlap_; /** * Start offsets of each tile in the user buffer, per attribute per tile. */ size_t** start_offsets_; /** Number of tiles in the tile slab. */ int64_t tile_num_; /** Used in calculations of tile ids. */ int64_t* tile_offset_per_dim_; }; /** The state for a tile slab copy. */ struct TileSlabState { /** Keeps track of whether a tile slab copy for an attribute id done. */ bool* copy_tile_slab_done_; /** Current coordinates in tile slab per attribute. */ void** current_coords_; /** * The offset in the local buffers of the next cell slab to be copied per * attribute. Note that this applies only to fixed-sized attributes * because the offsets of the variable-sized attributes can be derived from * the buffers that hold the fixed-sized offsets. */ size_t* current_offsets_; /** The current tile per attribute. */ int64_t* current_tile_; }; /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param array The array this array sorted read state belongs to. */ ArraySortedWriteState(Array* array); /** Destructor. */ ~ArraySortedWriteState(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns true if the current slab is finished being copied. */ bool copy_tile_slab_done() const; /** True if write is done for all attributes. */ bool done() const; /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Initializes the array sorted write state. * * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int init(); /** * Same as Array::write(), but it sorts the cells in the buffers based on the * array global cell order, prior to writing them to the disk. Note that this * function will fail if there is not enough system memory to hold the cells * of a 'tile slab' overlapping with the selected subarray. * * @param buffers The buffers that hold the input cells to be written. * @param buffer_sizes The corresponding buffer sizes. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int write(const void** buffers, const size_t* buffer_sizes); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** Function for advancing a cell slab during a copy operation. */ void *(*advance_cell_slab_) (void*); /** Counter for the AIO requests. */ size_t aio_cnt_; /** The AIO mutex conditions (one for each buffer). */ pthread_cond_t aio_cond_[2]; /** Data for the AIO requests. */ ASWS_Data aio_data_[2]; /** The current id of the buffers the next AIO will occur into. */ int aio_id_; /** The AIO mutex. */ pthread_mutex_t aio_mtx_; /** AIO requests. */ AIO_Request aio_request_[2]; /** The status of the AIO requests.*/ int aio_status_[2]; /** The thread that handles all the AIO in the background. */ pthread_t aio_thread_; /** True if the AIO thread is canceled. */ volatile bool aio_thread_canceled_; /** True if the AIO thread is running. */ volatile bool aio_thread_running_; /** The array this sorted read state belongs to. */ Array* array_; /** The ids of the attributes the array was initialized with. */ const std::vector attribute_ids_; /** * The sizes of the attributes. For variable-length attributes, sizeof(size_t) * is stored. */ std::vector attribute_sizes_; /** Number of allocated buffers. */ int buffer_num_; /** The user buffer offsets. */ size_t* buffer_offsets_; /** The user buffer sizes. */ const size_t* buffer_sizes_; /** The user buffers. */ const void** buffers_; /** Function for calculating cell slab info during a copy operation. */ void *(*calculate_cell_slab_info_) (void*); /** Function for calculating tile slab info during a copy operation. */ void *(*calculate_tile_slab_info_) (void*); /** The coordinates size of the array. */ size_t coords_size_; /** The copy mutex conditions (one for each buffer). */ pthread_cond_t copy_cond_[2]; /** The current id of the buffers the next copy will occur from. */ int copy_id_; /** The copy state, one per tile slab. */ CopyState copy_state_; /** The copy mutex. */ pthread_mutex_t copy_mtx_; /** The number of dimensions in the array. */ int dim_num_; /** The expanded subarray, such that it coincides with tile boundaries. */ void* expanded_subarray_; /** The query subarray. */ void* subarray_; /** Auxiliary variable used in calculate_tile_slab_info(). */ void* tile_coords_; /** Auxiliary variable used in calculate_tile_slab_info(). */ void* tile_domain_; /** The tile slab to be read for the first and second buffers. */ void* tile_slab_[2]; /** Indicates if the tile slab has been initialized. */ bool tile_slab_init_[2]; /** Normalized tile slab. */ void* tile_slab_norm_[2]; /** The info for each of the two tile slabs under investigation. */ TileSlabInfo tile_slab_info_[2]; /** The state for the current tile slab being copied. */ TileSlabState tile_slab_state_; /** Wait for copy flags, one for each local buffer. */ bool wait_copy_[2]; /** Wait for AIO flags, one for each local buffer. */ bool wait_aio_[2]; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Advances a cell slab focusing on column-major order, and updates * the CopyState and TileSlabState. * Used in copy_tile_slab(). * * @tparam T The domain type. * @param data Essentially a pointer to a ASWS_Data object. * @return void */ template static void *advance_cell_slab_col_s(void* data); /** * Advances a cell slab focusing on row-major order, and updates * the CopyState and TileSlabState. * Used in copy_tile_slab(). * * @tparam T The domain type. * @param data Essentially a pointer to a ASWS_Data object. * @return void */ template static void *advance_cell_slab_row_s(void* data); /** * Advances a cell slab when the requested order is column-major. * * @tparam T The domain type. * @param aid The id of the attribute in attribute_ids_ to focus on. * @return void */ template void advance_cell_slab_col(int aid); /** * Advances a cell slab when the requested order is row-major. * * @tparam T The domain type. * @param aid The id of the attribute in attribute_ids_ to focus on. * @return void */ template void advance_cell_slab_row(int aid); /** * Called when an AIO completes. * * @param data A ASWS_Request object. * @return void */ static void *aio_done(void* data); /** Sets the flag of wait_aio_[id] to true. */ void block_aio(int id); /** Sets the flag of wait_copy_[id] to true. */ void block_copy(int id); /** * Calculates the number of buffers to be allocated, based on the number * of attributes initialized for the array. * * @return void */ void calculate_buffer_num(); /** * Calculates the buffer sizes based on the array type. * * @return void */ void calculate_buffer_sizes(); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param data Essentially a pointer to a ASWS_Data object. * @return void */ template static void *calculate_cell_slab_info_col_col_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param data Essentially a pointer to a ASWS_Data object. * @return void */ template static void *calculate_cell_slab_info_col_row_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order in row-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param data Essentially a pointer to a AWRS_Data object. * @return void */ template static void *calculate_cell_slab_info_row_col_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is row-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param data Essentially a pointer to a AWRS_Data object. * @return void */ template static void *calculate_cell_slab_info_row_row_s(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_col_col(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is column-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_col_row(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is row-major and the **array** cell * order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_row_row(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **user** cell order is row-major and the **array** cell * order is column-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_row_col(int id, int64_t tid); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **array** cell order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @param tid The tile id. * @return void. */ template void calculate_cell_slab_info_row(int id, int64_t tid); /** * Calculates the **normalized** tile domain overlapped by the input tile * slab. Note that this domain is the same for all tile slabs * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_domain(int id); /** * Calculates the info used in the copy_tile_slab() function. * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_slab_info(int id); /** * Calculates tile slab info for the case where the **array** tile order is * column-major * * @tparam T The domain type. * @param data Essentially a pointer to a ASWS_Data object. * @return void */ template static void *calculate_tile_slab_info_col(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **array** tile order is column-major. * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_slab_info_col(int id); /** * Calculates tile slab info for the case where the **array** tile order is * row-major * * @tparam T The domain type. * @param data Essentially a pointer to a ASWS_Data object. * @return void */ template static void *calculate_tile_slab_info_row(void* data); /** * Calculates the info used in the copy_tile_slab() function, for the case * where the **array** tile order is row-major. * * @tparam T The domain type. * @param id The tile slab id. * @return void. */ template void calculate_tile_slab_info_row(int id); /** * Function called by the AIO thread. * * @param context This is practically the ArraySortedWriteState object for * which the function is called (typically *this* is passed to this * argument by the caller). */ static void *aio_handler(void* context); /** * Copies a tile slab from the user buffers into the local buffers, * properly re-organizing the cell order to follow the array global * cell order. * * @return void. */ void copy_tile_slab(); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order, * focusing on a particular fixed-length attribute. * * @tparam T The attribute type. * @param aid The index on attribute_ids_ to focus on. * @param bid The index on the copy state buffers to focus on. * @return void. */ template void copy_tile_slab(int aid, int bid); /** * Copies a tile slab from the local buffers into the user buffers, * properly re-organizing the cell order to fit the targeted order, * focusing on a particular variable-length attribute. * * @tparam T The attribute type. * @param aid The index on attribute_ids_ to focus on. * @param bid The index on the copy state buffers to focus on. * @return void. */ template void copy_tile_slab_var(int aid, int bid); /** * Creates the copy state buffers. * * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int create_copy_state_buffers(); /** * Creates the user buffers. * * @param buffers The user buffers that hold the input cells to be written. * @param buffer_sizes The corresponding buffer sizes. * @return void */ void create_user_buffers(const void** buffers, const size_t* buffer_sizes); /** * Fills the **entire** buffer of the current copy tile slab with the input id * with empty values, based on the template type. Applicable only to * fixed-sized attributes. * * @tparam T The attribute type. * @param bid The buffer id corresponding to the targeted attribute. * @return void */ template void fill_with_empty(int bid); /** * Fills the **a single** cell in a variable-sized buffer of the current copy * tile slab with the input id with an empty value, based on the template t * ype. Applicable only to variable-sized attributes. * * @tparam T The attribute type. * @param bid The buffer id corresponding to the targeted attribute. * @return void */ template void fill_with_empty_var(int bid); /** Frees the copy state. */ void free_copy_state(); /** Frees the tile slab info. */ void free_tile_slab_info(); /** Frees the tile slab state. */ void free_tile_slab_state(); /** * Returns the cell id along the **array** order for the current coordinates * in the tile slab state for a particular attribute. * * @tparam T The domain type. * @param aid The targeted attribute. * @return The cell id. */ template int64_t get_cell_id(int aid); /** * Returns the tile id along the **array** order for the current coordinates * in the tile slab state for a particular attribute. * * @tparam T The domain type. * @param aid The targeted attribute. * @return The tile id. */ template int64_t get_tile_id(int aid); /** * Handles the AIO requests. * * @tparam T The domain type. * @return void. */ template void handle_aio_requests(); /** Initializes the AIO requests. */ void init_aio_requests(); /** Initializes the copy state. */ void init_copy_state(); /** Initializes the tile slab info. */ void init_tile_slab_info(); /** * Initializes the tile slab info for a particular tile slab, using the * input tile number. * * @tparam T The domain type. * @param id The slab id. * @return void. */ template void init_tile_slab_info(int id); /** Initializes the tile slab state. */ void init_tile_slab_state(); /** * Locks the AIO mutex. * * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int lock_aio_mtx(); /** * Locks the copy mutex. * * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int lock_copy_mtx(); /** * Retrieves the next column tile slab to be processed. * * @tparam T The domain type. * @return True if the next tile slab was retrieved, and false otherwise. */ template bool next_tile_slab_col(); /** * Retrieves the next row tile slab to be processed. * * @tparam T The domain type. * @return True if the next tile slab was retrieved, and false otherwise. */ template bool next_tile_slab_row(); /** * Same as Array::write(), but it sorts the cells in the buffers based on the * global cell order prior to writing them on disk. Note that this function * will fail if there is not enough system memory to hold the cells of a * 'tile slab' overlapping with the selected subarray. * * @tparam T The domain type. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ template int write(); /** * Same as write(), but the cells are provided by the user sorted in * column-major order with respect to the selected subarray. * * @tparam T The domain type. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ template int write_sorted_col(); /** * Same as write(), but the cells are provided by the user sorted in * row-major order with respect to the selected subarray. * * @tparam T The domain type. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ template int write_sorted_row(); /** * Signals an AIO condition. * * @param id The id of the AIO condition to be signaled. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int release_aio(int id); /** * Signals a copy condition. * * @param id The id of the copy condition to be signaled. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int release_copy(int id); /** Resets the copy state for the current copy id. */ void reset_copy_state(); /** * Resets the tile_coords_ auxiliary variable. * * @tparam T The domain type. * @return void. */ template void reset_tile_coords(); /** * Resets the tile slab state. * * @tparam T The domain type. * @return void. */ template void reset_tile_slab_state(); /** * Sends an AIO request. * * @param aio_id The id of the tile slab the AIO request focuses on. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int send_aio_request(int aio_id); /** * Unlocks the AIO mutex. * * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int unlock_aio_mtx(); /** * Unlocks the copy mutex. * * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int unlock_copy_mtx(); /** * Calculates the new tile and local buffer offset for the new (already * computed) current cell coordinates in the tile slab. * * @param aid The attribute id to focus on. * @return void. */ void update_current_tile_and_offset(int aid); /** * Calculates the new tile and local buffer offset for the new (already * computed) current cell coordinates in the tile slab. * * @tparam T The domain type * @param aid The attribute id to focus on. * @return void. */ template void update_current_tile_and_offset(int aid); /** * Waits on a copy operation for the buffer with input id to finish. * * @param id The id of the buffer the copy operation must be completed. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int wait_copy(int id); /** * Waits on a AIO operation for the buffer with input id to finish. * * @param id The id of the buffer the AIO operation must be completed. * @return TILEDB_ASWS_OK for success and TILEDB_ASWS_ERR for error. */ int wait_aio(int id); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/c_api/000077500000000000000000000000001453617025200217735ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/c_api/tiledb.h000066400000000000000000001605261453617025200234210ustar00rootroot00000000000000/** * @file tiledb.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file declares the C API for TileDB. */ #ifndef __TILEDB_H__ #define __TILEDB_H__ #include "tiledb_constants.h" #ifdef HAVE_MPI #include #endif #include #include #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /** Default error message. */ #define TILEDB_ERRMSG std::string("[TileDB] Error: ") /** Maximum error message length. */ #define TILEDB_ERRMSG_MAX_LEN 2000 /* ********************************* */ /* MACROS */ /* ********************************* */ #ifdef __cplusplus extern "C" { #endif /**@{*/ /** C Library export. */ #define TILEDB_EXPORT __attribute__((visibility("default"))) /**@}*/ /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern char tiledb_errmsg[TILEDB_ERRMSG_MAX_LEN]; /* ********************************* */ /* CONFIG */ /* ********************************* */ /** Used to pass congiguration parameters to TileDB. */ typedef struct TileDB_Config { /** * The TileDB home directory. If it is set to "" (empty string) or NULL, the * default home directory will be used, which is ~/.tiledb/. */ const char* home_; #ifdef HAVE_MPI /** The MPI communicator. Use NULL if no MPI is used. */ MPI_Comm* mpi_comm_; #endif /** * The method for reading data from a file. * It can be one of the following: * - TILEDB_IO_MMAP (default) * TileDB will use mmap. * - TILEDB_IO_READ * TileDB will use standard OS read. * - TILEDB_IO_MPI * TileDB will use MPI-IO read. */ int read_method_; /** * The method for writing data to a file. * It can be one of the following: * - TILEDB_IO_WRITE (default) * TileDB will use standard OS write. * - TILEDB_IO_MPI * TileDB will use MPI-IO write. */ int write_method_; /** * Allows some optimizations like disable file locking and keeping file handles open until explicitly closed * to help minimize excessive writes and other unpredictable behavior on shared posix systems like NFS and Lustre. * These can be overridden with env TILEDB_DISABLE_FILE_LOCKING and TILEDB_KEEP_FILE_HANDLES_OPEN. */ bool enable_shared_posixfs_optimizations_; TileDB_Config() { memset(this, 0, sizeof(struct TileDB_Config)); } } TileDB_Config; /* ********************************* */ /* CONTEXT */ /* ********************************* */ /** The TileDB context, which maintains state for the TileDB modules. */ typedef struct TileDB_CTX TileDB_CTX; /** * Initializes the TileDB context. * * @param tiledb_ctx The TileDB context to be initialized. * @param tiledb_config TileDB configuration parameters. If it is NULL, * TileDB will use its default configuration parameters. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_ctx_init( TileDB_CTX** tiledb_ctx, const TileDB_Config* tiledb_config); /** * Finalizes the TileDB context, properly freeing-up memory. * * @param tiledb_ctx The TileDB context to be finalized. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_ctx_finalize(TileDB_CTX* tiledb_ctx); /* ********************************* */ /* WORKSPACE */ /* ********************************* */ /** * Creates a new TileDB workspace. * * @param tiledb_ctx The TileDB context. * @param workspace The directory of the workspace to be created in the file * system. This directory should not be inside another TileDB workspace, * group, array or metadata directory. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_workspace_create( const TileDB_CTX* tiledb_ctx, const char* workspace); /* ********************************* */ /* GROUP */ /* ********************************* */ /** * Creates a new TileDB group. * * @param tiledb_ctx The TileDB context. * @param group The directory of the group to be created in the file system. * This should be a directory whose parent is a TileDB workspace or another * TileDB group. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_group_create( const TileDB_CTX* tiledb_ctx, const char* group); /* ********************************* */ /* ARRAY */ /* ********************************* */ /** A TileDB array object. */ typedef struct TileDB_Array TileDB_Array; /** The array schema. */ typedef struct TileDB_ArraySchema { /** * The TileDB workspace. It is a directory. */ char* array_workspace_; /** * The array name. It is a directory, whose parent must be a TileDB workspace, * or group. */ char* array_name_; /** The attribute names. */ char** attributes_; /** The number of attributes. */ int attribute_num_; /** * The tile capacity for the case of sparse fragments. If it is <=0, * TileDB will use its default. */ int64_t capacity_; /** * The cell order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR * - TILEDB_HILBERT. */ int cell_order_; /** * Specifies the number of values per attribute for a cell. If it is NULL, * then each attribute has a single value per cell. If for some attribute * the number of values is variable (e.g., in the case off strings), then * TILEDB_VAR_NUM must be used. */ int* cell_val_num_; /** * The compression type for each attribute (plus one extra at the end for the * coordinates). It can be one of the following: * - TILEDB_NO_COMPRESSION * - TILEDB_GZIP * - TILEDB_ZSTD * - TILEDB_LZ4 * - TILEDB_BLOSC * - TILEDB_BLOSC_LZ4 * - TILEDB_BLOSC_LZ4HC * - TILEDB_BLOSC_SNAPPY * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE * * If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all * attributes. */ int* compression_; /** Specifies the compression level */ int* compression_level_; /** Compression type for the offsets for variable number of cells TILEDB_VAR_NUM for attribute */ int* offsets_compression_; /** Compression level for the offsets for variable number of cells TILEDB_VAR_NUM for attribute */ int* offsets_compression_level_; /** * Specifies if the array is dense (1) or sparse (0). If the array is dense, * then the user must specify tile extents (see below). */ int dense_; /** The dimension names. */ char** dimensions_; /** The number of dimensions. */ int dim_num_; /** * The array domain. It should contain one [low, high] pair per dimension. * The type of the values stored in this buffer should match the coordinates * type. */ void* domain_; /** * The tile extents. There should be one value for each dimension. The type of * the values stored in this buffer should match the coordinates type. It * can be NULL only for sparse arrays. */ void* tile_extents_; /** * The tile order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR. */ int tile_order_; /** * The attribute types, plus an extra one in the end for the coordinates. * The attribute type can be one of the following: * - TILEDB_INT32, TILEDB_UINT32 * - TILEDB_INT64, TILEDB_UINT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR, TILEDB_INT8, TILEDB_UINT8 * - TILEDB_INT16, TILEDB_UINT16 * * The coordinate type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 */ int* types_; } TileDB_ArraySchema; /** * Populates a TileDB array schema object. * * @param tiledb_array_schema The array schema to be populated. * @param array_name The array name. * @param attributes The attribute names. * @param attribute_num The number of attributes. * @param capacity The tile capacity. * @param cell_order The cell order. * @param cell_val_num The number of values per attribute per cell. * @param compression The compression type for each attribute (plus an extra one * in the end for the coordinates). * @param compression_level The compression level associated with the compression type per attribute * @param dense Specifies if the array is dense (1) or sparse (0). * @param dimensions The dimension names. * @param dim_num The number of dimensions. * @param domain The array domain. * @param domain_len The length of *domain* in bytes. * @param tile_extents The tile extents. * @param tile_extents_len The length of *tile_extents* in bytes. * @param tile_order The tile order. * @param types The attribute types (plus one in the end for the coordinates). * @return TILEDB_OK for success and TILEDB_ERR for error. * @see TileDB_ArraySchema */ TILEDB_EXPORT int tiledb_array_set_schema( TileDB_ArraySchema* tiledb_array_schema, const char* array_name, const char** attributes, int attribute_num, int64_t capacity, int cell_order, const int* cell_val_num, const int* compression, const int* compression_level, const int* offsets_compression, const int* offsets_compression_level, int dense, const char** dimensions, int dim_num, const void* domain, size_t domain_len, const void* tile_extents, size_t tile_extents_len, int tile_order, const int* types); /** * Creates a new TileDB array. * * @param tiledb_ctx The TileDB context. * @param tiledb_array_schema The array schema. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_create( const TileDB_CTX* tiledb_ctx, const TileDB_ArraySchema* tiledb_array_schema); /** * Initializes a TileDB array. * * @param tiledb_ctx The TileDB context. * @param tiledb_array The array object to be initialized. The function * will allocate memory space for it. * @param array The directory of the array to be initialized. * @param mode The mode of the array. It must be one of the following: * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_SORTED_COL * - TILEDB_ARRAY_WRITE_SORTED_ROW * - TILEDB_ARRAY_WRITE_UNSORTED * - TILEDB_ARRAY_READ * - TILEDB_ARRAY_READ_SORTED_COL * - TILEDB_ARRAY_READ_SORTED_ROW * @param subarray The subarray in which the array read/write will be * constrained on. It should be a sequence of [low, high] pairs (one * pair per dimension), whose type should be the same as that of the * coordinates. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. * @param attributes A subset of the array attributes the read/write will be * constrained on. Note that the coordinates have special attribute name * TILEDB_COORDS. A NULL value indicates **all** attributes (including * the coordinates as the last attribute in the case of sparse arrays). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_init( const TileDB_CTX* tiledb_ctx, TileDB_Array** tiledb_array, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num); /** * Apply filter to the initialized array. This is useful * when the array is used for reading, and the user wishes to constrain the * read results with an expression. * * @param tiledb_array The TileDB array. * @param filter_expression An expression string that evaluates to a boolean * to allow for cells to be filtered out from the buffers while reading. * If NULL or empty, no filter is applied. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_apply_filter( const TileDB_Array* tiledb_array, const char *filter_expression); /** * Resets the subarray used upon initialization of the array. This is useful * when the array is used for reading, and the user wishes to change the * query subarray without having to finalize and re-initialize the array. * * @param tiledb_array The TileDB array. * @param subarray The new subarray. It should be a sequence of [low, high] * pairs (one pair per dimension), whose type should be the same as that of * the coordinates. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_reset_subarray( const TileDB_Array* tiledb_array, const void* subarray); /** * Resets the attributes used upon initialization of the array. * * @param tiledb_array The TileDB array. * @param attributes The new attributes to focus on. If it is NULL, then * all the attributes are used (including the coordinates in the case of * sparse arrays, or sparse writes to dense arrays). * @param attribute_num The number of the attributes. If *attributes* is NULL, * then this should be 0. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_reset_attributes( const TileDB_Array* tiledb_array, const char** attributes, int attribute_num); /** * Retrieves the schema of an already initialized array. * * @param tiledb_array The TileDB array object (must already be initialized). * @param tiledb_array_schema The array schema to be retrieved. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_get_schema( const TileDB_Array* tiledb_array, TileDB_ArraySchema* tiledb_array_schema); /** * Retrieves the schema of an array from disk. * * @param tiledb_ctx The TileDB context. * @param array The directory of the array whose schema will be retrieved. * @param tiledb_array_schema The array schema to be retrieved. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_load_schema( const TileDB_CTX* tiledb_ctx, const char* array, TileDB_ArraySchema* tiledb_array_schema); /** * Frees the input array schema struct, properly deallocating memory space. * * @param tiledb_array_schema The array schema to be freed. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_free_schema( TileDB_ArraySchema* tiledb_array_schema); /** * Performs a write operation to an array. * The array must be initialized in one of the following write modes, * each of which has a different behaviour: * - TILEDB_ARRAY_WRITE: \n * In this mode, the cell values are provided in the buffers respecting * the cell order on the disk (specified in the array schema). It is * practically an **append** operation, * where the provided cell values are simply written at the end of * their corresponding attribute files. This mode leads to the best * performance. The user may invoke this function an arbitrary number * of times, and all the writes will occur in the same fragment. * Moreover, the buffers need not be synchronized, i.e., some buffers * may have more cells than others when the function is invoked. * - TILEDB_ARRAY_WRITE_SORTED_COL: \n * In this mode, the cell values are provided in the buffer in column-major * order with respect to the subarray used upon array initialization. * TileDB will properly re-organize the cells so that they follow the * array cell order for storage on the disk. * - TILEDB_ARRAY_WRITE_SORTED_ROW: \n * In this mode, the cell values are provided in the buffer in row-major * order with respect to the subarray used upon array initialization. * TileDB will properly re-organize the cells so that they follow the * array cell order for storage on the disk. * - TILEDB_ARRAY_WRITE_UNSORTED: \n * This mode is applicable to sparse arrays, or when writing sparse updates * to a dense array. One of the buffers holds the coordinates. The cells * in this mode are given in an arbitrary, unsorted order (i.e., without * respecting how the cells must be stored on the disk according to the * array schema definition). Each invocation of this function internally * sorts the cells and writes them to the disk in the proper order. In * addition, each invocation creates a **new** fragment. Finally, the * buffers in each invocation must be synchronized, i.e., they must have * the same number of cell values across all attributes. * * @param tiledb_array The TileDB array object (must be already initialized). * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attribute order specified in * tiledb_array_init() or tiledb_array_reset_attributes(). The case of * variable-sized attributes is special. Instead of providing a single * buffer for such an attribute, **two** must be provided: the second * holds the variable-sized cell values, whereas the first holds the * start offsets of each cell in the second buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there should * be a one-to-one correspondence). * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_write( const TileDB_Array* tiledb_array, const void** buffers, const size_t* buffer_sizes); /** * Performs a read operation on an array. * The array must be initialized in one of the following read modes, * each of which has a different behaviour: * - TILEDB_ARRAY_READ: \n * In this mode, the cell values are stored in the buffers respecting * the cell order on the disk (specified in the array schema). This mode * leads to the best performance. * - TILEDB_ARRAY_READ_SORTED_COL: \n * In this mode, the cell values are stored in the buffers in column-major * order with respect to the subarray used upon array initialization. * - TILEDB_ARRAY_READ_SORTED_ROW: \n * In this mode, the cell values are stored in the buffer in row-major * order with respect to the subarray used upon array initialization. * * @param tiledb_array The TileDB array. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * tiledb_array_init() or tiledb_array_reset_attributes(). The case of * variable-sized attributes is special. Instead of providing a single * buffer for such an attribute, **two** must be provided: the second * will hold the variable-sized cell values, whereas the first holds the * start offsets of each cell in the second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the input * buffers (there is a one-to-one correspondence). The function will attempt * to write as many results as can fit in the buffers, and potentially * alter the buffer size to indicate the size of the *useful* data written * in the buffer. If a buffer cannot hold all results, the function will * still succeed, writing as much data as it can and turning on an overflow * flag which can be checked with function tiledb_array_overflow(). The * next invocation will resume from the point the previous one stopped, * without inflicting a considerable performance penalty due to overflow. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_read( const TileDB_Array* tiledb_array, void** buffers, size_t* buffer_sizes); /** * Identical to tiledb_array_read, but skips N cells for each attribute * before reading data into the buffer. An example where this is useful is * as follows - user queries fields/attributes [ F0, F1, F2 ], but is only * interested in cells where F0 > C. So, a good query would read blocks of F0 * data, determine which cells are needed and would only fetch data from F1 * and F2 for the required cells. This can be achieved by skipping over the * discarded cells * @params - identical to tiledb_array_read() * @param skip_counts The number of cells to skip over for each buffer/attribute. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_array_skip_and_read( const TileDB_Array* tiledb_array, void** buffers, size_t* buffer_sizes, size_t* skip_counts); /** * Evaluates the cell based on the filter expression applied to the tiledb array. * @param tiledb_array The TileDB array * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second will hold the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). * @param positions The position of the cell in the buffer to be evaluated. * There should be one position for each of the input buffers. * @return true(1) or false(0) for successful evaluation and TILEDB_ERR(-1) otherwise. * The onus is on the client to check if there was an error during * evaluation when TILEB_ERR is returned. */ TILEDB_EXPORT int tiledb_array_evaluate_cell( const TileDB_Array* tiledb_array, void** buffers, size_t* buffer_sizes, int64_t* positions); /** * Checks if a read operation for a particular attribute resulted in a * buffer overflow. * * @param tiledb_array The TileDB array. * @param attribute_id The id of the attribute for which the overflow is * checked. This id corresponds to the position of the attribute name * placed in the *attributes* input of tiledb_array_init(), or * tiledb_array_reset_attributes() (the positions start from 0). * If *attributes* was NULL in the * above functions, then the attribute id corresponds to the order * in which the attributes were defined in the array schema upon the * array creation. Note that, in that case, the extra coordinates * attribute corresponds to the last extra attribute, i.e., its id * is *attribute_num*. * @return TILEDB_ERR for error, 1 for overflow, and 0 otherwise. */ TILEDB_EXPORT int tiledb_array_overflow( const TileDB_Array* tiledb_array, int attribute_id); /** * Consolidates the fragments of an array into a single fragment. * * @param tiledb_ctx The TileDB context. * @param array The name of the TileDB array to be consolidated. * @param buffer_size (Optional) The size of buffers for reading/writing attributes during consolidation. Default is 10M. * @param batch size (Optional) When specified, consolidation will occur batch-wise with a smaller batch_size set of * fragments getting consolidating together. Default is all fragments. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_consolidate( const TileDB_CTX* tiledb_ctx, const char* array, size_t buffer_size = TILEDB_CONSOLIDATION_BUFFER_SIZE, int batch_size = -1); /** * Finalizes a TileDB array, properly freeing its memory space. * * @param tiledb_array The array to be finalized. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_finalize( TileDB_Array* tiledb_array); /** * Syncs all currently written files in the input array. * * @param tiledb_array The array to be synced. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_sync( TileDB_Array* tiledb_array); /** * Syncs the currently written files associated with the input attribute * in the input array. * * @param tiledb_array The array to be synced. * @param attribute The name of the attribute to be synced. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_sync_attribute( TileDB_Array* tiledb_array, const char* attribute); /** A TileDB array iterator. */ typedef struct TileDB_ArrayIterator TileDB_ArrayIterator; /** * Initializes an array iterator for reading cells, potentially constraining it * on a subset of attributes, as well as a subarray. The cells will be read * in the order they are stored on the disk, maximizing performance. * * @param tiledb_ctx The TileDB context. * @param tiledb_array_it The TileDB array iterator to be created. The function * will allocate the appropriate memory space for the iterator. * @param array The directory of the array the iterator is initialized for. * @param mode The read mode, which can be one of the following: * - TILEDB_ARRAY_READ\n * Reads the cells in the native order they are stored on the disk. * - TILEDB_ARRAY_READ_SORTED_COL\n * Reads the cells in column-major order within the specified subarray. * - TILEDB_ARRAY_READ_SORTED_ROW\n * Reads the cells in column-major order within the specified subarray. * @param subarray The subarray in which the array iterator will be * constrained on. It should be a sequence of [low, high] pairs (one * pair per dimension), whose type should be the same as that of the * coordinates. If it is NULL, then the subarray is set to the entire * array domain. * @param attributes A subset of the array attributes the iterator will be * constrained on. Note that the coordinates have special attribute name * TILEDB_COORDS. A NULL value indicates **all** attributes (including * the coordinates as the last attribute in the case of sparse arrays). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param buffers This is an array of buffers similar to tiledb_array_read(). * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read cells. The iterator will * read from the disk the relevant cells in batches, by fitting as many * cell values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding size (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many cells as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_iterator_init( const TileDB_CTX* tiledb_ctx, TileDB_ArrayIterator** tiledb_array_it, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes); /** * Initializes an array iterator for reading cells, potentially constraining it * on a subset of attributes, as well as a subarray. The cells will be read * in the order they are stored on the disk, maximizing performance. * * @param tiledb_ctx The TileDB context. * @param tiledb_array_it The TileDB array iterator to be created. The function * will allocate the appropriate memory space for the iterator. * @param array The directory of the array the iterator is initialized for. * @param mode The read mode, which can be one of the following: * - TILEDB_ARRAY_READ\n * Reads the cells in the native order they are stored on the disk. * - TILEDB_ARRAY_READ_SORTED_COL\n * Reads the cells in column-major order within the specified subarray. * - TILEDB_ARRAY_READ_SORTED_ROW\n * Reads the cells in column-major order within the specified subarray. * @param subarray The subarray in which the array iterator will be * constrained on. It should be a sequence of [low, high] pairs (one * pair per dimension), whose type should be the same as that of the * coordinates. If it is NULL, then the subarray is set to the entire * array domain. * @param attributes A subset of the array attributes the iterator will be * constrained on. Note that the coordinates have special attribute name * TILEDB_COORDS. A NULL value indicates **all** attributes (including * the coordinates as the last attribute in the case of sparse arrays). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param buffers This is an array of buffers similar to tiledb_array_read(). * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read cells. The iterator will * read from the disk the relevant cells in batches, by fitting as many * cell values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding size (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many cells as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @param filter_expression An expression string that evaluates to a boolean * to allow for cells to be filtered out from the buffers while reading. * If NULL or empty, no filter is applied. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_iterator_init_with_filter( const TileDB_CTX* tiledb_ctx, TileDB_ArrayIterator** tiledb_array_it, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes, const char* filter_expression); /** * Resets the subarray used upon initialization of the iterator. This is useful * when the array is used for reading, and the user wishes to change the * query subarray without having to finalize and re-initialize the array. * * @param tiledb_array_it The TileDB iterator. * @param subarray The new subarray. It should be a sequence of [low, high] * pairs (one pair per dimension), whose type should be the same as that of * the coordinates. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_iterator_reset_subarray( TileDB_ArrayIterator* tiledb_array_it, const void* subarray); /** * Retrieves the current cell value for a particular attribute. * * @param tiledb_array_it The TileDB array iterator. * @param attribute_id The id of the attribute for which the cell value * is retrieved. This id corresponds to the position of the attribute name * placed in the *attributes* input of tiledb_array_iterator_init() * (the positions start from 0). * If *attributes* was NULL in the above function, then the attribute id * corresponds to the order in which the attributes were defined in the * array schema upon the array creation. Note that, in that case, the extra * coordinates attribute corresponds to the last extra attribute, i.e., * its id is *attribute_num*. * @param value The cell value to be retrieved. Note that its type is the * same as that defined in the array schema for the corresponding attribute. * Note also that the function essentially returns a pointer to this value * in the internal buffers of the iterator. * @param value_size The size (in bytes) of the retrieved value. Useful mainly * for the case of variable-sized cells. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_iterator_get_value( TileDB_ArrayIterator* tiledb_array_it, int attribute_id, const void** value, size_t* value_size); /** * Advances the iterator by one cell. * * @param tiledb_array_it The TileDB array iterator. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_iterator_next( TileDB_ArrayIterator* tiledb_array_it); /** * Checks if the the iterator has reached its end. * * @param tiledb_array_it The TileDB array iterator. * @return TILEDB_ERR for error, 1 for having reached the end, and 0 otherwise. */ TILEDB_EXPORT int tiledb_array_iterator_end( TileDB_ArrayIterator* tiledb_array_it); /** * Finalizes an array iterator, properly freeing the allocating memory space. * * @param tiledb_array_it The TileDB array iterator to be finalized. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_array_iterator_finalize( TileDB_ArrayIterator* tiledb_array_it); /* ********************************* */ /* METADATA */ /* ********************************* */ /** Specifies the metadata schema. */ typedef struct TileDB_MetadataSchema { /** * The metadata name. It is a directory, whose parent must be a TileDB * workspace, group, or array. */ char* metadata_name_; /** The attribute names. */ char** attributes_; /** The number of attributes. */ int attribute_num_; /** * The tile capacity. If it is <=0, TileDB will use its default. */ int64_t capacity_; /** * Specifies the number of values per attribute for a cell. If it is NULL, * then each attribute has a single value per cell. If for some attribute * the number of values is variable (e.g., in the case off strings), then * TILEDB_VAR_NUM must be used. */ int* cell_val_num_; /** * The compression type for each attribute (plus one extra at the end for the * key). It can be one of the following: * - TILEDB_NO_COMPRESSION * - TILEDB_GZIP * - TILEDB_ZSTD * - TILEDB_LZ4 * - TILEDB_BLOSC * - TILEDB_BLOSC_LZ4 * - TILEDB_BLOSC_LZ4HC * - TILEDB_BLOSC_SNAPPY * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE * * If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all * attributes. */ int* compression_; int* compression_level_; /** * The attribute types. * The attribute type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR. */ int* types_; } TileDB_MetadataSchema; /** A TileDB metadata object. */ typedef struct TileDB_Metadata TileDB_Metadata; /** * Populates a TileDB metadata schema object. * * @param tiledb_metadata_schema The metadata schema C API struct. * @param metadata_name The metadata name. * @param attributes The attribute names. * @param attribute_num The number of attributes. * @param capacity The tile capacity. * @param cell_val_num The number of values per attribute per cell. * @param compression The compression type for each attribute (plus an extra one * in the end for the key). * @param types The attribute types. * @return TILEDB_OK for success and TILEDB_ERR for error. * @see TileDB_MetadataSchema */ TILEDB_EXPORT int tiledb_metadata_set_schema( TileDB_MetadataSchema* tiledb_metadata_schema, const char* metadata_name, const char** attributes, int attribute_num, int64_t capacity, const int* cell_val_num, const int* compression, const int* compression_level, const int* types); /** * Creates a new TileDB metadata object. * * @param tiledb_ctx The TileDB context. * @param metadata_schema The metadata schema. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_metadata_create( const TileDB_CTX* tiledb_ctx, const TileDB_MetadataSchema* metadata_schema); /** * Initializes a TileDB metadata object. * * @param tiledb_ctx The TileDB context. * @param tiledb_metadata The metadata object to be initialized. The function * will allocate memory space for it. * @param metadata The directory of the metadata to be initialized. * @param mode The mode of the metadata. It must be one of the following: * - TILEDB_METADATA_WRITE * - TILEDB_METADATA_READ * @param attributes A subset of the metadata attributes the read/write will be * constrained on. Note that the keys have a special attribute name * called TILEDB_KEYS. A NULL value indicates **all** attributes (including * the keys as an extra attribute in the end). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_init( const TileDB_CTX* tiledb_ctx, TileDB_Metadata** tiledb_metadata, const char* metadata, int mode, const char** attributes, int attribute_num); /** * Resets the attributes used upon initialization of the metadata. * * @param tiledb_metadata The TileDB metadata. * @param attributes The new attributes to focus on. If it is NULL, then * all the attributes are used. * @param attribute_num The number of the attributes. If *attributes* is NULL, * then this should be 0. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_reset_attributes( const TileDB_Metadata* tiledb_metadata, const char** attributes, int attribute_num); /** * Retrieves the schema of an already initialized metadata object. * * @param tiledb_metadata The TileDB metadata object (must already be * initialized). * @param tiledb_metadata_schema The metadata schema to be retrieved. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_metadata_get_schema( const TileDB_Metadata* tiledb_metadata, TileDB_MetadataSchema* tiledb_metadata_schema); /** * Retrieves the schema of a metadata object from disk. * * @param tiledb_ctx The TileDB context. * @param metadata The directory of the metadata whose schema will be retrieved. * @param tiledb_metadata_schema The metadata schema to be retrieved. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_metadata_load_schema( const TileDB_CTX* tiledb_ctx, const char* metadata, TileDB_MetadataSchema* tiledb_metadata_schema); /** * Frees the input metadata schema struct, properly deallocating memory space. * * @param tiledb_metadata_schema The metadata schema to be freed. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_metadata_free_schema( TileDB_MetadataSchema* tiledb_metadata_schema); /** * Performs a write operation to a metadata object. The metadata must be * initialized with mode TILEDB_METADATA_WRITE. This function behave very * similarly to tiledb_array_write() when the array is initialized with mode * TILEDB_ARRAY_WRITE_UNSORTED. * * @param tiledb_metadata The TileDB metadata (must be already initialized). * @param keys The buffer holding the metadata keys. These keys must be * strings, serialized one after the other in the *keys* buffer. * @param keys_size The size (in bytes) of buffer *keys*. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * tiledb_metadata_init() or tiledb_metadata_reset_attributes(). The case of * variable-sized attributes is special. Instead of providing a single * buffer for such an attribute, **two** must be provided: the second * holds the variable-sized values, whereas the first holds the * start offsets of each value in the second buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_metadata_write( const TileDB_Metadata* tiledb_metadata, const char* keys, size_t keys_size, const void** buffers, const size_t* buffer_sizes); /** * Performs a read operation on a metadata object, which must be initialized * with mode TILEDB_METADATA_READ. The read is performed on a single key. * * @param tiledb_metadata The TileDB metadata. * @param key The query key, which must be a string. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * tiledb_metadata_init() or tiledb_metadata_reset_attributes(). The case of * variable-sized attributes is special. Instead of providing a single * buffer for such an attribute, **two** must be provided: the second * will hold the variable-sized values, whereas the first holds the * start offsets of each value in the second buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the input * buffers (there should be a one-to-one correspondence). The function will * attempt to write the value corresponding to the key, and potentially * alter the respective size in *buffer_sizes* to indicate the *useful* * data written. If a buffer cannot * hold the result, the function will still succeed, turning on an overflow * flag which can be checked with function tiledb_metadata_overflow(). * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_metadata_read( const TileDB_Metadata* tiledb_metadata, const char* key, void** buffers, size_t* buffer_sizes); /** * Checks if a read operation for a particular attribute resulted in a * buffer overflow. * * @param tiledb_metadata The TileDB metadata. * @param attribute_id The id of the attribute for which the overflow is * checked. This id corresponds to the position of the attribute name * placed in the *attributes* input of tiledb_metadata_init(), or * tiledb_metadata_reset_attributes(). The positions start from 0. * If *attributes* was NULL in the * above functions, then the attribute id corresponds to the order * in which the attributes were defined in the metadata schema upon the * metadata creation. * @return TILEDB_ERR for error, 1 for overflow, and 0 otherwise. */ TILEDB_EXPORT int tiledb_metadata_overflow( const TileDB_Metadata* tiledb_metadata, int attribute_id); /** * Consolidates the fragments of a metadata object into a single fragment. * * @param tiledb_ctx The TileDB context. * @param metadata The name of the TileDB metadata to be consolidated. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_consolidate( const TileDB_CTX* tiledb_ctx, const char* metadata); /** * Finalizes a TileDB metadata object, properly freeing the memory space. * * @param tiledb_metadata The metadata to be finalized. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_finalize( TileDB_Metadata* tiledb_metadata); /** A TileDB metadata iterator. */ typedef struct TileDB_MetadataIterator TileDB_MetadataIterator; /** * Initializes a metadata iterator, potentially constraining it * on a subset of attributes. The values will be read in the order they are * stored on the disk (which is random), maximizing performance. * * @param tiledb_ctx The TileDB context. * @param tiledb_metadata_it The TileDB metadata iterator to be created. The * function will allocate the appropriate memory space for the iterator. * @param metadata The directory of the metadata the iterator is initialized * for. * @param attributes A subset of the metadata attributes the iterator will be * constrained on. Note that the keys have a special value called * TILEDB_KEYS. A NULL value indicates **all** attributes. * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param buffers This is an array of buffers similar to tiledb_metadata_read(). * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read values. The iterator will * read from the disk the values in batches, by fitting as many * values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding sizes (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many values as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_iterator_init( const TileDB_CTX* tiledb_ctx, TileDB_MetadataIterator** tiledb_metadata_it, const char* metadata, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes); /** * Retrieves the current value for a particular attribute. * * @param tiledb_metadata_it The TileDB metadata iterator. * @param attribute_id The id of the attribute for which the overflow is * checked. This id corresponds to the position of the attribute name * placed in the *attributes* input of tiledb_metadata_init(), or * tiledb_metadata_reset_attributes(). The positions start from 0. * If *attributes* was NULL in the * above functions, then the attribute id corresponds to the order * in which the attributes were defined in the metadata schema upon the * metadata creation. * @param value The value to be retrieved. Note that its type is the * same as that defined in the metadata schema. Note also that the function * returns a pointer to this value in the internal buffers of the iterator. * @param value_size The size (in bytes) of the retrieved value. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_iterator_get_value( TileDB_MetadataIterator* tiledb_metadata_it, int attribute_id, const void** value, size_t* value_size); /** * Advances the iterator by one position. * * @param tiledb_metadata_it The TileDB metadata iterator. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_iterator_next( TileDB_MetadataIterator* tiledb_metadata_it); /** * Checks if the the iterator has reached its end. * * @param tiledb_metadata_it The TileDB metadata iterator. * @return TILEDB_ERR for error, 1 for having reached the end, and 0 otherwise. */ TILEDB_EXPORT int tiledb_metadata_iterator_end( TileDB_MetadataIterator* tiledb_metadata_it); /** * Finalizes the iterator, properly freeing the allocating memory space. * * @param tiledb_metadata_it The TileDB metadata iterator. * @return TILEDB_OK on success, and TILEDB_ERR on error. */ TILEDB_EXPORT int tiledb_metadata_iterator_finalize( TileDB_MetadataIterator* tiledb_metadata_it); /* ********************************* */ /* DIRECTORY MANAGEMENT */ /* ********************************* */ /** * Clears a TileDB directory. The corresponding TileDB object (workspace, * group, array, or metadata) will still exist after the execution of the * function, but it will be empty (i.e., as if it was just created). * * @param tiledb_ctx The TileDB context. * @param dir The TileDB directory to be cleared. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_clear( const TileDB_CTX* tiledb_ctx, const char* dir); /** * Deletes a TileDB directory (workspace, group, array, or metadata) entirely. * * @param tiledb_ctx The TileDB context. * @param dir The TileDB directory to be deleted. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_delete( const TileDB_CTX* tiledb_ctx, const char* dir); /** * Moves a TileDB directory (workspace, group, array or metadata). * * @param tiledb_ctx The TileDB context. * @param old_dir The old TileDB directory. * @param new_dir The new TileDB directory. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_move( const TileDB_CTX* tiledb_ctx, const char* old_dir, const char* new_dir); /** * Lists all TileDB workspaces, copying their directory names in the input * string buffers. * * @param tiledb_ctx The TileDB context. * @param workspaces An array of strings that will store the listed workspaces. * Note that this should be pre-allocated by the user. If the size of * each string is smaller than the corresponding workspace path name, the * function will probably segfault. It is a good idea to allocate to each * workspace string TILEDB_NAME_MAX_LEN characters. * @param workspace_num The number of allocated elements of the *workspaces* * string array. After the function execution, this will hold the actual * number of workspaces written in the *workspaces* string array. If the * number of allocated elements is smaller than the number of existing * TileDB workspaces, the function will return an error. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_ls_workspaces( const TileDB_CTX* tiledb_ctx, const char* parent_dir, char** workspaces, int* workspace_num); /** * Counts the number of TileDB workspaces. * * @param tiledb_ctx The TileDB context. * @param workspace_num The number of TileDB workspaces to be returned. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_ls_workspaces_c( const TileDB_CTX* tiledb_ctx, const char* parent_dir, int* workspace_num); /** * Lists all the TileDB objects in a directory, copying their names into the * input string buffers. * * @param tiledb_ctx The TileDB context. * @param parent_dir The parent directory of the TileDB objects to be listed. * @param dirs An array of strings that will store the listed TileDB objects. * Note that the user is responsible for allocating the appropriate memory * space for this array of strings. A good idea is to allocate for each * string TILEDB_NAME_MAX_LEN characters. * @param dir_types The types of the corresponding TileDB objects in *dirs*, * which can be the following: * - TILEDB_WORKSPACE * - TILEDB_GROUP * - TILEDB_ARRAY * - TILEDB_METADATA * @param dir_num The number of elements allocated by the user for *dirs*. After * the function terminates, this will hold the actual number of TileDB * objects that were stored in *dirs*. If the number of * allocated elements is smaller than the number of existing TileDB objects * in the parent directory, the function will return an error. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_ls( const TileDB_CTX* tiledb_ctx, const char* parent_dir, char** dirs, int* dir_types, int* dir_num); /** * Counts the TileDB objects in a directory. * * @param tiledb_ctx The TileDB context. * @param parent_dir The parent directory of the TileDB objects to be listed. * @param dir_num The number of TileDB objects to be returned. * @return TILEDB_OK for success and TILEDB_ERR for error. */ TILEDB_EXPORT int tiledb_ls_c( const TileDB_CTX* tiledb_ctx, const char* parent_dir, int* dir_num); /* ********************************* */ /* ASYNCHRONOUS I/O (AIO) */ /* ********************************* */ /** Describes an AIO (read or write) request. */ typedef struct TileDB_AIO_Request { /** * An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * tiledb_array_init() or tiledb_array_reset_attributes(). The case of * variable-sized attributes is special. Instead of providing a single * buffer for such an attribute, **two** must be provided: the second * holds the variable-sized cell values, whereas the first holds the * start offsets of each cell in the second buffer. */ void** buffers_; /** * The sizes (in bytes) allocated by the user for the * buffers (there is a one-to-one correspondence). In the case of reads, * the function will attempt * to write as many results as can fit in the buffers, and potentially * alter the buffer sizes to indicate the size of the *useful* data written * in the corresponding buffers. */ size_t* buffer_sizes_; /** Function to be called upon completion of the request. */ void *(*completion_handle_) (void*); /** Data to be passed to the completion handle. */ void* completion_data_; /** * Applicable only to read requests. * Indicates whether a buffer has overflowed during a read request. * If it is NULL, it will be ignored. Otherwise, it must be an array * with as many elements as the number of attributes specified in * tiledb_array_init() or tiledb_array_reset_attributes(). */ bool* overflow_; /** * The status of the AIO request. It can be one of the following: * - TILEDB_AIO_COMPLETED * The request is completed. * - TILEDB_AIO_INPROGRESS * The request is still in progress. * - TILEDB_AIO_OVERFLOW * At least one of the input buffers overflowed (applicable only to AIO * read requests) * - TILEDB_AIO_ERR * The request caused an error (and thus was canceled). */ int status_; /** * The subarray in which the array read/write will be * constrained on. It should be a sequence of [low, high] pairs (one * pair per dimension), whose type should be the same as that of the * coordinates. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. */ const void* subarray_; } TileDB_AIO_Request; /** * Issues an asynchronous read request. * * @param tiledb_array An initialized TileDB array. * @param tiledb_aio_request An asynchronous read request. * @return TILEDB_OK upon success, and TILEDB_ERR upon error. * * @note If the same input request is in progress, the function will fail. * Moreover, if the input request was issued in the past and caused an * overflow, the new call will resume it IF there was no other request * in between the two separate calls for the same input request. * In other words, a new request that is different than the previous * one resets the internal read state. */ TILEDB_EXPORT int tiledb_array_aio_read( const TileDB_Array* tiledb_array, TileDB_AIO_Request* tiledb_aio_request); /** * Issues an asynchronous write request. * * @param tiledb_array An initialized TileDB array. * @param tiledb_aio_request An asynchronous write request. * @return TILEDB_OK upon success, and TILEDB_ERR upon error. */ TILEDB_EXPORT int tiledb_array_aio_write( const TileDB_Array* tiledb_array, TileDB_AIO_Request* tiledb_aio_request); #undef TILEDB_EXPORT #ifdef __cplusplus } #endif #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/c_api/tiledb_constants.h000066400000000000000000000172231453617025200255100ustar00rootroot00000000000000/** * @file tiledb_constants.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file contains global constants. */ #ifndef __TILEDB_CONSTANTS_H__ #define __TILEDB_CONSTANTS_H__ #include #include #include /** Version. */ #define TILEDB_VERSION "0.5.2" /**@{*/ /** Return code. */ #define TILEDB_ERR -1 #define TILEDB_OK 0 /**@}*/ /**@{*/ /** Array mode. */ #define TILEDB_ARRAY_READ 0 #define TILEDB_ARRAY_READ_SORTED_COL 1 #define TILEDB_ARRAY_READ_SORTED_ROW 2 #define TILEDB_ARRAY_WRITE 3 #define TILEDB_ARRAY_WRITE_SORTED_COL 4 #define TILEDB_ARRAY_WRITE_SORTED_ROW 5 #define TILEDB_ARRAY_WRITE_UNSORTED 6 #define TILEDB_ARRAY_CONSOLIDATE 7 /**@}*/ /**@{*/ /** Metadata mode. */ #define TILEDB_METADATA_READ 0 #define TILEDB_METADATA_WRITE 1 /**@}*/ /**@{*/ /** I/O method. */ #define TILEDB_IO_MMAP 0 #define TILEDB_IO_READ 1 #define TILEDB_IO_MPI 2 #define TILEDB_IO_WRITE 0 /**@}*/ /**@{*/ /** Asynchronous I/O (AIO) code. */ #define TILEDB_AIO_ERR -1 #define TILEDB_AIO_COMPLETED 0 #define TILEDB_AIO_INPROGRESS 1 #define TILEDB_AIO_OVERFLOW 2 /**@}*/ /** * The TileDB home directory, where TileDB-related system metadata structures * are kept. If it is set to "", then the home directory is set to "~/.tiledb" * by default. */ #define TILEDB_HOME "" /**@{*/ /** TileDB object type. */ #define TILEDB_WORKSPACE 0 #define TILEDB_GROUP 1 #define TILEDB_ARRAY 2 #define TILEDB_METADATA 3 /**@}*/ /** The maximum length for the names of TileDB objects. */ #define TILEDB_NAME_MAX_LEN 4096 /** Size of the buffer used during consolidation. */ #define TILEDB_CONSOLIDATION_BUFFER_SIZE 10000000 // ~10 MB /**@{*/ /** Special empty cell value. */ #define TILEDB_EMPTY_CHAR CHAR_MAX #define TILEDB_EMPTY_INT8 INT8_MAX #define TILEDB_EMPTY_INT16 INT16_MAX #define TILEDB_EMPTY_INT32 INT32_MAX #define TILEDB_EMPTY_INT64 INT64_MAX #define TILEDB_EMPTY_UINT8 UINT8_MAX #define TILEDB_EMPTY_UINT16 UINT16_MAX #define TILEDB_EMPTY_UINT32 UINT32_MAX #define TILEDB_EMPTY_UINT64 UINT64_MAX #define TILEDB_EMPTY_FLOAT32 FLT_MAX #define TILEDB_EMPTY_FLOAT64 DBL_MAX /**@}*/ /**@{*/ /** Special value indicating a variable number or size. */ #define TILEDB_VAR_NUM INT_MAX #define TILEDB_VAR_SIZE (size_t)-1 /**@}*/ /**@{*/ /** Data type. */ #define TILEDB_INT32 0 #define TILEDB_INT64 1 #define TILEDB_FLOAT32 2 #define TILEDB_FLOAT64 3 #define TILEDB_CHAR 4 #define TILEDB_INT8 5 #define TILEDB_UINT8 6 #define TILEDB_INT16 7 #define TILEDB_UINT16 8 #define TILEDB_UINT32 9 #define TILEDB_UINT64 10 /**@}*/ /**@{*/ /** Tile or cell order. */ #define TILEDB_ROW_MAJOR 0 #define TILEDB_COL_MAJOR 1 #define TILEDB_HILBERT 2 /**@}*/ /**@{*/ /** Compression type. Range from 0-15 */ #define TILEDB_NO_COMPRESSION 0 #define TILEDB_GZIP 1 #define TILEDB_ZSTD 2 #define TILEDB_LZ4 3 #define TILEDB_BLOSC 4 #define TILEDB_BLOSC_LZ4 5 #define TILEDB_BLOSC_LZ4HC 6 #define TILEDB_BLOSC_SNAPPY 7 #define TILEDB_BLOSC_ZLIB 8 #define TILEDB_BLOSC_ZSTD 9 #define TILEDB_RLE 10 /**@}*/ /**@{*/ /** Pre compression filter defines. Can only range from 1 to 3 for now */ #define TILEDB_DELTA_ENCODE (1<<4) #define TILEDB_BIT_SHUFFLE (2<<4) /**@}*/ /**@{*/ /** Post compression filter defines. Can only range from 1 to 3 for now */ #define TILEDB_POST_FILTER_EXAMPLE_UNUSED (1<<6) /**@}*/ /**@{*/ /** Special attribute name. */ #define TILEDB_COORDS "__coords" #define TILEDB_KEY "__key" /**@}*/ /**@{*/ /** Special TileDB file name suffix. */ #define TILEDB_FILE_SUFFIX ".tdb" #define TILEDB_GZIP_SUFFIX ".gz" /**@}*/ /** Chunk size in GZIP decompression. */ #define TILEDB_GZIP_CHUNK_SIZE 131072 // 128KB /**@{*/ /** Special TileDB file name. */ #define TILEDB_ARRAY_SCHEMA_FILENAME "__array_schema.tdb" #define TILEDB_METADATA_SCHEMA_FILENAME "__metadata_schema.tdb" #define TILEDB_BOOK_KEEPING_FILENAME "__book_keeping" #define TILEDB_FRAGMENT_FILENAME "__tiledb_fragment.tdb" #define TILEDB_GROUP_FILENAME "__tiledb_group.tdb" #define TILEDB_WORKSPACE_FILENAME "__tiledb_workspace.tdb" /**@}*/ /**@{*/ /** Size of buffer used for sorting. */ #define TILEDB_SORTED_BUFFER_SIZE 10000000 // ~10MB #define TILEDB_SORTED_BUFFER_VAR_SIZE 10000000 // ~10MB /**@}*/ /**@{*/ /** Compression levels. */ #ifndef TILEDB_COMPRESSION_LEVEL_GZIP # define TILEDB_COMPRESSION_LEVEL_GZIP Z_DEFAULT_COMPRESSION #endif #ifndef TILEDB_COMPRESSION_LEVEL_ZSTD # define TILEDB_COMPRESSION_LEVEL_ZSTD 1 #endif #ifndef TILEDB_COMPRESSION_LEVEL_BLOSC # define TILEDB_COMPRESSION_LEVEL_BLOSC 5 #endif #ifndef TILEDB_COMPRESSION_LEVEL_LZ4 #define TILEDB_COMPRESSION_LEVEL_LZ4 1 #endif /**@}*/ #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/c_api/tiledb_storage.h000066400000000000000000000173361453617025200251450ustar00rootroot00000000000000/** * @file tiledb_storage.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 Omics Data Automation Inc. and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Storage Prototypes to expose some of the filesystem functionality implemented in TileDB. * The implementation is in tiledb.cc. * */ #ifndef __TILEDB_STORAGE_H__ #define __TILEDB_STORAGE_H__ #include "tiledb.h" #include #include /** * Checks if the input directory is a workspace. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if the directory is a workspace, and *false* otherwise. */ bool is_workspace(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Checks if the input directory is a group. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if the directory is a group, and *false* otherwise. */ bool is_group(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Checks if the input directory is an array. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if the directory is an array, and *false* otherwise. */ bool is_array(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Checks if the input directory is a fragment. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if the directory is a fragment, and *false* otherwise. */ bool is_fragment(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Checks if the input directory is a metadata object. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if the directory is a metadata object, and *false* otherwise. */ bool is_metadata(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Checks if the input is an existing directory. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if *dir* is an existing directory, and *false* otherwise. */ bool is_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Returns absolute path for given path only for posix filesystems. * For cloud data stores, this is a noop and returns the given path. * * @param tiledb_ctx TileDB Context * @param dir The directory to be checked. * @return *true* if *dir* is an existing directory, and *false* otherwise. */ std::string real_dir(const TileDB_CTX* tiledb_ctx, const std::string& dirpath); /** * Checks if the input is an existing file. * * @param tiledb_ctx TileDB Context * @param file The file to be checked. * @return *true* if *file* is an existing file, and *false* otherwise. */ bool is_file(const TileDB_CTX* tiledb_ctx, const std::string& file); /** * Return parent directory for given path * * @param path * @return parent of given path or NULL */ std::string parent_dir(const std::string& path); /** */ std::string parent_dir(const std::string& path); /** * Return current working directory * * @param tiledb_ctx TileDB Context * return current working directory if it can be determined or empty string. */ std::string current_working_dir(const TileDB_CTX* tiledb_ctx); /** * Set working directory to dir * * @param tiledb_ctx TileDB Context * @param dir The directory to be set as working dir * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int set_working_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Creates a new directory. * * @param tiledb_ctx TileDB Context * @param dir The name of the directory to be created. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int create_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Deletes a directory. Note that the directory must not contain other * directories, but it should only contain files. * * @param tiledb_ctx TileDB Context * @param dirname The name of the directory to be deleted. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int delete_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** Returns the names of the directories inside the input directory. * * @param tiledb_ctx TileDB Context * @param dir The input directory. * @return The vector of directories contained in the input directory. */ std::vector get_dirs(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** Returns the names of the files inside the input directory. * * @param tiledb_ctx TileDB Context * @param dir The input directory. * @return The vector of directories contained in the input directory. */ std::vector get_files(const TileDB_CTX* tiledb_ctx, const std::string& dir); /** * Returns the size of the input file. * * @param tiledb_ctx TileDB Context * @param filename The name of the file whose size is to be retrieved. * @return The file size on success, and TILEDB_UT_ERR for error. */ ssize_t file_size(const TileDB_CTX* tiledb_ctx, const std::string& file); int create_file(const TileDB_CTX* tiledb_ctx, const std::string& filename, int flags, mode_t mode); /** * Reads data from a file into a buffer. * * @param tiledb_ctx TileDB Context * @param filename The name of the file. * @param offset The offset in the file from which the read will start. * @param buffer The buffer into which the data will be written. * @param length The size of the data to be read from the file. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int read_file(const TileDB_CTX* tiledb_ctx, const std::string& filename, off_t offset, void *buffer, size_t length); /** * Writes the input buffer to a file. * * @param tiledb_ctx TileDB Context * @param filename The name of the file. * @param buffer The input buffer. * @param buffer_size The size of the input buffer. * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int write_file(const TileDB_CTX* tiledb_ctx, const std::string& filename, const void *buffer, size_t buffer_size); /** * Deletes a file from the filesystem * * @param tiledb_ctx TileDB Context * @param filename The name of the file to be deleted. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int delete_file(const TileDB_CTX* tiledb_ctx, const std::string& filename); /** * Closes any open file handles associated with a file. If the file does not exist, * or if there are no open file handles it is a noop). * * @param tiledb_ctx TileDB Context * @param filename The name of the file. * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int close_file(const TileDB_CTX* tiledb_ctx, const std::string& filename); #endif /* __TILEDB_STORAGE_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/c_api/tiledb_utils.h000066400000000000000000000072741453617025200246410ustar00rootroot00000000000000/** * @file tiledb_utils.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 Omics Data Automation Inc. and Intel Corporation * @copyright Copyright (c) 2020-2021 Omics Data Automation Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Utility prototypes to access TileDB functionality in an opaque, convenient fashion. * */ #ifndef __TILEDB_UTILS_H__ #define __TILEDB_UTILS_H__ #include "tiledb.h" #include #include namespace TileDBUtils { bool is_cloud_path(const std::string& path); std::string get_path(const std::string &path); std::string append_path(const std::string& dir, const std::string& path); int initialize_workspace(TileDB_CTX **ptiledb_ctx, const std::string& workspace, const bool overwrite=false, const bool enable_shared_posixfs_optimizations=false); int create_workspace(const std::string &workspace, bool replace=false); bool workspace_exists(const std::string& workspace); bool array_exists(const std::string& workspace, const std::string& array_name); std::vector get_array_names(const std::string& workspace); std::vector get_fragment_names(const std::string& workspace); bool is_dir(const std::string& dirpath); std::string real_dir(const std::string& dirpath); int create_dir(const std::string& dirpath); int delete_dir(const std::string& dirpath); bool is_file(const std::string& filepath); std::vector get_dirs(const std::string& dirpath); std::vector get_files(const std::string& dirpath); ssize_t file_size(const std::string& filepath); void print_md5_hash(unsigned char* buffer, size_t length); /** * buffer is malloc'ed and has to be freed by calling function */ int read_entire_file(const std::string& filename, void **buffer, size_t *length); int read_file(const std::string& filename, off_t offset, void *buffer, size_t length); int write_file(const std::string& filename, const void *buffer, size_t length, const bool overwrite=false); int delete_file(const std::string& filename); int move_across_filesystems(const std::string& src, const std::string& dest); int create_temp_filename(char *path, size_t path_length); /** * access to compression algorithms */ int create_codec(void **handle, int compression_type, int compression_level); int compress(void *handle, unsigned char* segment, size_t segment_size, void** compressed_segment, size_t& compressed_segment_size); int decompress(void *handle, unsigned char* compressed_segment, size_t compressed_segment_size, unsigned char* segment, size_t segment_size); void finalize_codec(void *handle); } #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/000077500000000000000000000000001453617025200217755ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/blosc_prototypes.h000066400000000000000000000010121453617025200255520ustar00rootroot00000000000000// Function Pointers for blosc #ifdef BLOSC_IMPL #undef BLOSC_DLL_EXPORT #define BLOSC_DLL_EXPORT __declspec(dllexport) #else #undef BLOSC_DLL_EXPORT #define BLOSC_DLL_EXPORT extern #endif BLOSC_DLL_EXPORT void (*blosc_init)(); BLOSC_DLL_EXPORT void (*blosc_destroy)(); BLOSC_DLL_EXPORT int (*blosc_set_compressor)(const char *); BLOSC_DLL_EXPORT int (*blosc_compress)(int, int, size_t, size_t, const void *, void *, size_t); BLOSC_DLL_EXPORT int (*blosc_decompress)(const void *, void *, size_t) genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec.h000066400000000000000000000155051453617025200232310ustar00rootroot00000000000000/** * @file codec.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the base class for compression and decompression. */ #ifndef __CODEC_H__ #define __CODEC_H__ #include "array_schema.h" #include "codec_filter.h" #include #include #include #include #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_CD_OK 0 #define TILEDB_CD_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_CD_ERRMSG std::string("[TileDB::Codec] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_cd_errmsg; /** Stores the state necessary when writing cells to a fragment. */ class Codec { public: /* ****************************** */ /* STATIC METHODS */ /* ****************************** */ typedef std::function create_fn_t; static int register_codec(int compression_type, Codec::create_fn_t create_fn); static bool is_registered_codec(int compression_type); static Codec::create_fn_t get_registered_codec(int compression_type); static Codec* create(const ArraySchema* array_schema, const int attribute_id, const bool is_offsets_compression=false); // Generalized non-TileDB codec usage static int create(void **handle, int compression_type, int compression_level); static int get_default_level(const int compression_type); static int print_errmsg(const std::string& msg); /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param compression_level */ Codec(int compression_level) { compression_level_ = compression_level; } /** * Destructor */ virtual ~Codec() { if (tile_compressed_ != NULL) { free(tile_compressed_); } if (pre_compression_filter_) { delete pre_compression_filter_; } if (post_compression_filter_) { delete post_compression_filter_; } } /* ********************************* */ /* MUTATORS */ /* ********************************* */ // Clears old error conditions void clear_dlerror() { dl_error_ = std::string(""); dlerror(); } void set_dlerror() { char *errmsg = dlerror(); if (errmsg) { dl_error_.empty()?(dl_error_=errmsg):(dl_error_+=std::string("\n")+errmsg); } } std::string& get_dlerror() { return dl_error_; } void *get_dlopen_handle(const std::string& name) { return get_dlopen_handle(name, ""); } void *get_dlopen_handle(const std::string& name, const std::string& version) { void *handle; std::string prefix("lib"); #ifdef __APPLE__ std::string suffix(".dylib"); #elif __linux__ std::string suffix(".so"); #else # error Platform not supported #endif clear_dlerror(); for (std::string dl_path : dl_paths_) { std::string path = dl_path+prefix+name; if (version.empty()) { handle = dlopen((path+suffix).c_str(), RTLD_GLOBAL|RTLD_NOW); } else { #ifdef __APPLE__ handle = dlopen((path+"."+version+suffix).c_str(), RTLD_GLOBAL|RTLD_NOW); #else handle = dlopen((path+suffix+"."+version).c_str(), RTLD_GLOBAL|RTLD_NOW); #endif } if (handle) { clear_dlerror(); return handle; } else { set_dlerror(); } } return handle; } #define BIND_SYMBOL(H, X, Y, Z) \ do { \ clear_dlerror(); \ X = Z dlsym(H, Y); \ if (!X) { \ set_dlerror(); \ throw std::system_error(ECANCELED, std::generic_category(), dl_error_); \ } \ } while (false) /** */ const std::string& name() { return name_; } int compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size); int decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size); /** * @return TILEDB_CD_OK on success and TILEDB_CD_ERR on error. */ virtual int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) = 0; /** * @return TILEDB_CD_OK on success and TILEDB_CD_ERR on error. */ virtual int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) = 0; void set_pre_compression(CodecFilter* filter) { pre_compression_filter_ = filter; }; void set_post_compression(CodecFilter* filter) { post_compression_filter_ = filter; }; /* ********************************* */ /* PROTECTED ATTRIBUTES */ /* ********************************* */ protected: std::string name_ = ""; int compression_level_; CodecFilter* pre_compression_filter_ = NULL; CodecFilter* post_compression_filter_ = NULL; /** Internal buffer used in the case of compression. */ void* tile_compressed_ = NULL; /** Allocated size for internal buffer used in the case of compression. */ size_t tile_compressed_allocated_size_ = 0; std::string dl_error_; #ifdef __APPLE__ std::vector dl_paths_ = {"/opt/homebrew/lib/", "/usr/local/Cellar/lib/", "/usr/local/lib/", "/usr/lib/", ""}; #elif __linux__ std::vector dl_paths_ = {"/usr/lib64/", "/usr/lib/", ""}; #else # error Platform not supported #endif }; #endif /*__CODEC_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_blosc.h000066400000000000000000000062731453617025200244150ustar00rootroot00000000000000/** * @file codec_blosc.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * CodecBlosc derived from Codec for Blosc support * */ #ifndef __CODEC_BLOSC_H__ #define __CODEC_BLOSC_H__ #include "codec.h" // Function Pointers for blosc #if !defined(BLOSC_EXTERN_DECL) # define BLOSC_EXTERN_DECL #endif BLOSC_EXTERN_DECL void (*blosc_init)(); BLOSC_EXTERN_DECL void (*blosc_destroy)(); BLOSC_EXTERN_DECL int (*blosc_set_compressor)(const char *); BLOSC_EXTERN_DECL int (*blosc_compress)(int, int, size_t, size_t, const void *, void *, size_t); BLOSC_EXTERN_DECL int (*blosc_decompress)(const void *, void *, size_t); class CodecBlosc : public Codec { public: CodecBlosc(int compression_level, std::string compressor, size_t type_size):Codec(compression_level) { static std::once_flag loaded; static void *dl_handle = NULL; compressor_ = compressor; type_size_ = type_size; std::call_once(loaded, [this]() { dl_handle = get_dlopen_handle("blosc"); if (dl_handle) { BIND_SYMBOL(dl_handle, blosc_init, "blosc_init", (void (*)())); BIND_SYMBOL(dl_handle, blosc_destroy, "blosc_destroy", (void (*)())); BIND_SYMBOL(dl_handle, blosc_set_compressor, "blosc_set_compressor", (int (*)(const char *))); BIND_SYMBOL(dl_handle, blosc_compress, "blosc_compress", (int (*)(int, int, size_t, size_t, const void *, void *, size_t))); BIND_SYMBOL(dl_handle, blosc_decompress, "blosc_decompress", (int (*)(const void *, void *, size_t))); } else { throw std::system_error(ECANCELED, std::generic_category(), dl_error_ + " Blosc library not found. Install Blosc and setup library paths."); } }); name_ = "Blosc"; } int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) override; int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) override; private: std::string compressor_; size_t type_size_; }; #endif /*__CODEC_BLOSC_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_filter.h000066400000000000000000000076761453617025200246100ustar00rootroot00000000000000/** * @file codec_filter.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the base class for pre and post compression filters */ #ifndef __CODEC_FILTER_H__ #define __CODEC_FILTER_H__ #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_CDF_OK 0 #define TILEDB_CDF_ERR -1 /**@}*/ /* ****************************** */ /* MACROS */ /* ****************************** */ /** Default error message. */ #define TILEDB_CDF_ERRMSG std::string("[TileDB::CodecFilter] Error: ") class CodecFilter { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param compression_level */ CodecFilter(int type, bool filter_in_place=true) { type_ = type; filter_in_place_ = filter_in_place; } virtual ~CodecFilter() { if (filter_buffer_ != NULL) { free(filter_buffer_); } } /** */ const std::string& name() { return filter_name_; } const bool in_place() { return filter_in_place_; } int allocate_buffer(size_t size) { if (filter_buffer_ == NULL) { assert(filter_buffer_allocated_size_ == 0); filter_buffer_ = malloc(size); filter_buffer_allocated_size_ = size; } else if (filter_buffer_allocated_size_ < size) { filter_buffer_ = realloc(filter_buffer_, size); filter_buffer_allocated_size_ = size; } if (filter_buffer_ == NULL) { return TILEDB_CDF_ERR; } else { return TILEDB_CDF_OK; } } int type() { return type_; } unsigned char* buffer() { return reinterpret_cast(filter_buffer_); } virtual int code(unsigned char* tile, size_t tile_size, unsigned char* tile_coded, size_t& tile_coded_size) { return print_errmsg("virtual method should be overridden"); } virtual int code(unsigned char *tile, size_t tile_size) { return print_errmsg("virtual method should be overridden"); } virtual int decode(unsigned char* tile_coded, size_t tile_coded_size, void** tile, size_t& tile_size) { return print_errmsg("virtual method should be overridden"); } virtual int decode(unsigned char* tile_coded, size_t tile_coded_size) { return print_errmsg("virtual method should be overridden"); } int print_errmsg(const std::string& msg); protected: std::string filter_name_ = ""; bool filter_in_place_; int type_; /** * Internal buffer malloc'ed if necessary * Support only for pre compression filters for now */ void* filter_buffer_ = NULL; size_t filter_buffer_allocated_size_ = 0; }; #endif /* __CODEC_FILTER_H_ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_filter_bit_shuffle.h000066400000000000000000000037101453617025200271430ustar00rootroot00000000000000/** * @file codec_filter_bit_shuffle.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the Bit Shuffle Pre-Compression Filter */ #ifndef __CODEC_BIT_SHUFFLE_H__ #define __CODEC_BIT_SHUFFLE_H__ #include "codec_filter.h" #include "tiledb_constants.h" class CodecBitShuffle : public CodecFilter { public: using CodecFilter::CodecFilter; CodecBitShuffle(int type): CodecFilter(type, false) { filter_name_ = "Bit Shuffle"; type_ = type; } /** * @return TILEDB_CDF_OK on success and TILEDB_CDF_ERR on error. */ int code(unsigned char* tile, size_t tile_size) override; /** * @return TILEDB_CDF_OK on success and TILEDB_CDF_ERR on error. */ int decode(unsigned char* tile_coded, size_t tile_coded_size) override; }; #endif /* __CODEC_BIT_SHUFFLE_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_filter_delta_encode.h000066400000000000000000000040501453617025200272550ustar00rootroot00000000000000/** * @file codec_filter_delta_encode.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the Delta Encoder Pre-Compression Filter */ #ifndef __CODEC_DELTA_ENCODING_H__ #define __CODEC_DELTA_ENCODING_H__ #include "codec_filter.h" #include "tiledb_constants.h" class CodecDeltaEncode : public CodecFilter { public: using CodecFilter::CodecFilter; CodecDeltaEncode(int type, int stride=1): CodecFilter(type, true) { filter_name_ = "Delta Encoding"; stride_ = stride; } const int stride() { return stride_; } /** * @return TILEDB_CDF_OK on success and TILEDB_CDF_ERR on error. */ int code(unsigned char* tile, size_t tile_size) override; /** * @return TILEDB_CDF_OK on success and TILEDB_CDF_ERR on error. */ int decode(unsigned char* tile, size_t tile_size) override; private: int stride_; }; #endif /* __CODEC_DELTA_ENCODING_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_gzip.h000066400000000000000000000034361453617025200242620ustar00rootroot00000000000000/** * @file codec_gzip.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * CodecGzip derived from Codec for gzip support * */ #ifndef __CODEC_GZIP_H__ #define __CODEC_GZIP_H__ #include "codec.h" class CodecGzip : public Codec { public: using Codec::Codec; CodecGzip(int compression_level):Codec(compression_level) { name_ = "GZIP"; } int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) override; int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) override; }; #endif /*__CODEC_GZIP_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_lz4.h000066400000000000000000000034251453617025200240200ustar00rootroot00000000000000/** * @file codec_lz4.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * CodecLZ4 derived from Codec for LZ4 support * */ #ifndef __CODEC_LZ4_H__ #define __CODEC_LZ4_H__ #include "codec.h" class CodecLZ4 : public Codec { public: using Codec::Codec; CodecLZ4(int compression_level):Codec(compression_level) { name_ = "LZ4"; } int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) override; int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) override; }; #endif /*__CODEC_LZ4_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_rle.h000066400000000000000000000040531453617025200240670ustar00rootroot00000000000000/** * @file codec_rle.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * CodecGzip derived from Codec for RLE support * */ #ifndef __CODEC_RLE_H__ #define __CODEC_RLE_H__ #include "codec.h" class CodecRLE : public Codec { public: CodecRLE(int attribute_num, int dim_num, int cell_order, bool is_coords, size_t value_size):Codec(0){ attribute_num_ = attribute_num; dim_num_ = dim_num; cell_order_ = cell_order; is_coords_ = is_coords; value_size_ = value_size; name_ = "RLE"; } int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) override; int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) override; private: int attribute_num_; int dim_num_; int cell_order_; bool is_coords_; size_t value_size_; }; #endif /*__CODEC_RLE_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/codec/codec_zstd.h000066400000000000000000000102531453617025200242700ustar00rootroot00000000000000/** * @file codec_zstd.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * CodecZstd derived from Codec for Zstandard support * */ #ifndef __CODEC_ZSTD_H__ #define __CODEC_ZSTD_H__ #include "codec.h" // Function Pointers for ZStd #if !defined(ZSTD_EXTERN_DECL) # define ZSTD_EXTERN_DECL #endif ZSTD_EXTERN_DECL size_t(*ZSTD_compressBound)(size_t); ZSTD_EXTERN_DECL int(*ZSTD_isError)(size_t); ZSTD_EXTERN_DECL char *(*ZSTD_getErrorName)(size_t); ZSTD_EXTERN_DECL int(*ZSTD_maxCLevel)(void); ZSTD_EXTERN_DECL size_t(*ZSTD_compress)(void *, size_t, const void *, size_t, int); ZSTD_EXTERN_DECL size_t(*ZSTD_decompress)(void *, size_t, const void *, size_t); ZSTD_EXTERN_DECL char *(*ZSTD_createCCtx)(void); ZSTD_EXTERN_DECL size_t(*ZSTD_freeCCtx)(char *); ZSTD_EXTERN_DECL size_t(*ZSTD_compressCCtx)(char *, void *, size_t, const void *, size_t, int); ZSTD_EXTERN_DECL char *(*ZSTD_createDCtx)(void); ZSTD_EXTERN_DECL size_t(*ZSTD_freeDCtx)(char *); ZSTD_EXTERN_DECL size_t(*ZSTD_decompressDCtx)(char *, void *, size_t, const void *, size_t); class CodecZStandard : public Codec { public: CodecZStandard(int compression_level):Codec(compression_level) { static std::once_flag loaded; static void *dl_handle = NULL; std::call_once(loaded, [this]() { dl_handle = get_dlopen_handle("zstd", "1"); if (dl_handle) { BIND_SYMBOL(dl_handle, ZSTD_compressBound, "ZSTD_compressBound", (size_t(*)(size_t))); BIND_SYMBOL(dl_handle, ZSTD_isError, "ZSTD_isError", (int(*)(size_t))); BIND_SYMBOL(dl_handle, ZSTD_getErrorName, "ZSTD_getErrorName", (char *(*)(size_t))); BIND_SYMBOL(dl_handle, ZSTD_maxCLevel, "ZSTD_maxCLevel", (int(*)(void))); BIND_SYMBOL(dl_handle, ZSTD_compress, "ZSTD_compress", (size_t(*)(void *, size_t, const void *, size_t, int))); BIND_SYMBOL(dl_handle, ZSTD_decompress, "ZSTD_decompress", (size_t(*)(void *, size_t, const void *, size_t))); BIND_SYMBOL(dl_handle, ZSTD_createCCtx, "ZSTD_createCCtx", (char*(*)(void))); BIND_SYMBOL(dl_handle, ZSTD_freeCCtx, "ZSTD_freeCCtx", (size_t(*)(char*))); BIND_SYMBOL(dl_handle, ZSTD_compressCCtx, "ZSTD_compressCCtx", (size_t(*)(char *, void *, size_t, const void *, size_t, int))); BIND_SYMBOL(dl_handle, ZSTD_createDCtx, "ZSTD_createDCtx", (char*(*)(void))); BIND_SYMBOL(dl_handle, ZSTD_freeDCtx, "ZSTD_freeDCtx", (size_t(*)(char*))); BIND_SYMBOL(dl_handle, ZSTD_decompressDCtx, "ZSTD_decompressDCtx", (size_t(*)(char *, void *, size_t, const void *, size_t))); } else { throw std::system_error(ECANCELED, std::generic_category(), dl_error_ + " ZStd library not found. Install ZStandard and/or setup library paths."); } }); name_ = "ZSTD"; } int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) override; int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) override; }; #endif /*__CODEC_ZSTD_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/expressions/000077500000000000000000000000001453617025200233025ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/expressions/expression.h000066400000000000000000000417041453617025200256600ustar00rootroot00000000000000/** * @file expression.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2019, 2022-2023 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the class Expression to store and process filters */ #ifndef __EXPRESSION_H__ #define __EXPRESSION_H__ #include "mpParser.h" #include "array_schema.h" #include /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_expr_errmsg; /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_EXPR_OK 0 #define TILEDB_EXPR_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_EXPR_ERRMSG std::string("[TileDB::Expression] Error: ") //Forward declaration class ArrayReadState; class Expression { public: /* ********************************* */ /* MUTATORS */ /* ********************************* */ Expression(std::string expression) : expression_(expression) {}; ~Expression() { delete parser_; } int init(const std::vector& attribute_ids, const ArraySchema* array_schema); void add_attribute(std::string name); int add_expression(std::string expression); int evaluate_cell(void **buffers, size_t* buffer_sizes, std::vector& positions); int evaluate_cell(void** buffers, size_t* buffer_sizes, int64_t* positions); /** * Only used for unit testing. */ int evaluate(void** buffers, size_t* buffer_sizes); private: void fixup_return_buffers(void** buffers, size_t* buffer_sizes, size_t number_of_cells, std::vector cells_to_be_dropped); std::string expression_; std::vector attributes_; const ArraySchema* array_schema_; bool is_initialized = false; mup::ParserX *parser_ = new mup::ParserX(mup::pckALL_NON_COMPLEX | mup::pckMATRIX); std::map attribute_map_; int coords_index_ = 0; int coords_index_in_buffer_ = 0; std::vector last_processed_buffer_index_; void assign_single_cell_value(const int attribute_id, void** buffers, const uint64_t buffer_index, const uint64_t position); void assign_fixed_cell_values(const int attribute_id, void** buffers, const uint64_t buffer_index, const uint64_t position); void assign_var_cell_values(const int attribute_id, void** buffers, size_t *buffer_sizes, const uint64_t buffer_index, const uint64_t position); inline const int get_cell_val_num(const std::string& attribute_name) const { if (attribute_name == TILEDB_COORDS) { return array_schema_->dim_num(); } else { return array_schema_->cell_val_num(array_schema_->attribute_id(attribute_name)); } } inline const size_t get_cell_size(const std::string& attribute_name) const { int attribute_id = array_schema_->attribute_id(attribute_name); int cell_val_num = get_cell_val_num(attribute_name); if (cell_val_num == TILEDB_VAR_NUM) { return sizeof(size_t); } else { return array_schema_->type_size(attribute_id)*cell_val_num; } } inline const size_t get_var_cell_type_size(const std::string& attribute_name) const { return array_schema_->type_size(array_schema_->attribute_id(attribute_name)); } inline void *offset_pointer(const std::string& attribute_name, const void* src, size_t offset) const { switch (get_var_cell_type_size(attribute_name)) { case 1: return (void *)((uint8_t *)(src) + offset); case 2: return (void *)((uint16_t *)(src) + offset); case 4: return (void *)((uint32_t *)(src) + offset); case 8: return (void *)((uint64_t *)(src) + offset); default: throw std::range_error("Attribute Type for " + attribute_name + " not supported in expressions"); } } }; /** * SplitCompare accepts 3 arguments * Input attribute name where the attributes is represented internally as string that is a list of strings * separated by some delimiter * Delimiter that is the ASCII integer value, note : muparserx does not accept characters as an arg * Comparison String that is compared with each token from the input string and returns true if there is a match * with any of the tokens, false otherwise. */ class SplitCompare : public mup::ICallback { public: SplitCompare():mup::ICallback(mup::cmFUNC, "splitcompare", 3){} void Eval(mup::ptr_val_type &ret, const mup::ptr_val_type *a_pArg, int a_iArgc) { mup::string_type input = a_pArg[0]->GetString(); mup::char_type delimiter = a_pArg[1]->GetInteger(); mup::string_type with = a_pArg[2]->GetString(); // The return type is boolean *ret = (mup::bool_type)false; std::stringstream ss(input); std::string word; while (!ss.eof()) { std::getline(ss, word, delimiter); if (word.compare(with) == 0) { *ret = (mup::bool_type)true; break; } } } const mup::char_type* GetDesc() const { return "splitcompare(input, delimiter, compare_string) - splitcompare tokenizes the string for the input attribute using the delimiter(specified as an ASCII integer) and then compares for any token match with the given string"; } mup::IToken* Clone() const { return new SplitCompare(*this); } }; /** * OprtSplitCompare, similar to SplitCompare above, but using a special operator "|=" where the LHS of the expression is the input * attribute name and the RHS is the string to be compared with. The delimiter is "|" for tokenizing the input string. */ class OprtSplitCompare : public mup::IOprtBin { public: OprtSplitCompare() : mup::IOprtBin(("|="), (int)mup::prRELATIONAL1, mup::oaLEFT) {} void Eval(mup::ptr_val_type &ret, const mup::ptr_val_type *a_pArg, int) { mup::string_type input = a_pArg[0]->GetString(); mup::char_type delimiter = '|'; mup::string_type with = a_pArg[1]->GetString(); // The return type is boolean *ret = (mup::bool_type)false; std::stringstream ss(input); std::string word; while (!ss.eof()) { std::getline(ss, word, delimiter); if (word.compare(with) == 0) { *ret = (mup::bool_type)true; break; } } } const mup::char_type* GetDesc() const { return " |= string - the operator tokenizes the attribute using the delimiter '|' and then looks for any token match with string"; } mup::IToken* Clone() const { return new OprtSplitCompare(*this); } }; #define PIPED_SEP '|' #define SLASHED_SEP '/' /** * Resolve accepts 3 arguments * Input attribute name where the attributes are represented internally as an array of integers separated * by a delimiter. Each input integer is compared against the input comparison strings. * First Comparison String which is compared with input integer if -eq 0 * Second Comparison String which is a delimited string, with the delimited position specified by the input integer if -gt 0 * Returns string of the same length as input attribute substituted with their character representations surmised * from the comparison strings */ class Resolve : public mup::ICallback { public: Resolve():mup::ICallback(mup::cmFUNC, "resolve", 3){} void Eval(mup::ptr_val_type &ret, const mup::ptr_val_type *a_pArg, int a_iArgc) { mup::matrix_type input = a_pArg[0]->GetArray(); mup::string_type cmp_str1 = a_pArg[1]->GetString(); mup::string_type cmp_str2 = a_pArg[2]->GetString(); // The return type is string mup::string_type ret_val; // A vector is represented as a matrix in muparserx with nCols=1 for (int i=0; i 0) { // ALT ret_val += get_segment(cmp_str2, val); } else if (val == 0) { // REF ret_val += cmp_str1; } else { // UNKNOWN ret_val += "."; } } } *ret = ret_val; } const mup::char_type* GetDesc() const { return "resolve(input, compare_string1, compare_string2) - the function works on a list of integers with optional delimiters '|' or '/' and compares against compare_string1 if the integer is 0 and with compare_string2 otherwise"; } mup::IToken* Clone() const { return new Resolve(*this); } private: std::string_view get_segment(std::string_view str, int segment_pos) { std::string::size_type pos; std::string::size_type next_pos = -1; for (int j=0; j vals; std::string::size_type pos = 0, next_pos; while ((next_pos = next(with, pos)) != std::string::npos) { vals.push_back(with.substr(pos, next_pos)); pos = next_pos+1; } vals.push_back(with.substr(pos)); pos = 0; while ((next_pos = next(input, pos)) != std::string::npos) { auto found = std::find(vals.begin(), vals.end(), input.substr(pos, next_pos)); if (found != vals.end()) { vals.erase(found); } else { return false; } pos = next_pos+1; } return (vals.size() == 1) && input.substr(pos) == vals[0]; } bool match_any(std::string_view input, std::string_view with) { std::string::size_type pos = 0, next_pos; while ((next_pos = next(input, pos)) != std::string::npos) { if (input.substr(pos, next_pos) == with) { return true; } pos = next_pos+1; } return input.substr(pos) == with; } std::string::size_type next(std::string_view str, std::string::size_type pos) { auto piped_pos = str.find(PIPED_SEP, pos); auto slashed_pos = str.find(SLASHED_SEP, pos); if (piped_pos < slashed_pos) { return piped_pos; } else { return slashed_pos; } } }; /** * IsHomRef accepts 1 argument * Input attribute name where the attributes are represented internally as an array of integers separated * by a delimiter. Each input integer is compared against the input comparison strings. * Returns true if all the integers ignoring delimiters are zero. */ class IsHomRef : public mup::ICallback { public: IsHomRef():mup::ICallback(mup::cmFUNC, "ishomref", 1){} void Eval(mup::ptr_val_type &ret, const mup::ptr_val_type *a_pArg, int a_iArgc) { mup::matrix_type input = a_pArg[0]->GetArray(); // The return type is boolean // A vector is represented as a matrix in muparserx with nCols=1 for (int i=0; iGetArray(); // The return type is boolean // A vector is represented as a matrix in muparserx with nCols=1 mup::int_type first_val = 0; for (int i=0; iGetArray(); // The return type is boolean *ret = (mup::bool_type)false; // input.GetRows() is zero for *unknown* values if (input.GetRows() > 0) { mup::int_type first_val = 0; for (int i=0; i #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_BK_OK 0 #define TILEDB_BK_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_BK_ERRMSG std::string("[TileDB::BookKeeping] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_bk_errmsg; /** Stores the book-keeping structures of a fragment. */ class BookKeeping { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param array_schema The array schema. * @param dense True if the fragment is dense, and false otherwise. * @param fragment_name The name of the fragment this book-keeping belongs to. * @param mode The mode in which the fragment was initialized in. */ BookKeeping( const ArraySchema* array_schema, bool dense, const std::string& fragment_name, int mode); /** Destructor. */ ~BookKeeping(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns the bounding coordinates. */ const std::vector& bounding_coords() const; /** Returns the number of cells in the tile at the input position. */ int64_t cell_num(int64_t tile_pos) const; /** * Returns ture if the corresponding fragment is dense, and false if it * is sparse. */ bool dense() const; /** Returns the (expanded) domain in which the fragment is constrained. */ const void* domain() const; /** Returns the number of cells in the last tile. */ int64_t last_tile_cell_num() const; /** Returns the MBRs. */ const std::vector& mbrs() const; /** Returns the non-empty domain in which the fragment is constrained. */ const void* non_empty_domain() const; /** Returns true if the array is in read mode. */ bool read_mode() const; /** Returns the number of tiles in the fragment. */ int64_t tile_num() const; /** Returns the tile offsets. */ const std::vector >& tile_offsets() const; /** Returns the variable tile offsets. */ const std::vector >& tile_var_offsets() const; /** Returns the variable tile sizes. */ const std::vector >& tile_var_sizes() const; /** Returns true if the array is in write mode. */ bool write_mode() const; /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Appends the tile bounding coordinates to the book-keeping structure. * * @param bounding_coords The bounding coordinates to be appended. * @return void */ void append_bounding_coords(const void* bounding_coords); /** * Appends the input MBR to the book-keeping structure. * * @param mbr The MBR to be appended. * @return void */ void append_mbr(const void* mbr); /** * Appends a tile offset for the input attribute. * * @param attribute_id The id of the attribute for which the offset is * appended. * @param step This is essentially the step by which the previous * offset will be expanded. It is practically the last tile size. * @return void */ void append_tile_offset(int attribute_id, size_t step); /** * Appends a variable tile offset for the input attribute. * * @param attribute_id The id of the attribute for which the offset is * appended. * @param step This is essentially the step by which the previous * offset will be expanded. It is practically the last variable tile size. * @return void */ void append_tile_var_offset(int attribute_id, size_t step); /** * Appends a variable tile size for the input attribute. * * @param attribute_id The id of the attribute for which the size is appended. * @param size The size to be appended. * @return void */ void append_tile_var_size(int attribute_id, size_t size); /** * Finalizes the book-keeping structures, properly flushing them to the disk. * @param fs The Storage File System class. * * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int finalize(StorageFS *fs); /** * Initializes the book-keeping structures. * * @param non_empty_domain The non-empty domain in which the array read/write * will be constrained. * @return TILEDB_BK_OK for success, and TILEDB_OK_ERR for error. */ int init(const void* non_empty_domain); /** * Loads the book-keeping structures from the disk. * @param fs The Storage File System class. * * @return TILEDB_BK_OK for success, and TILEDB_OK_ERR for error. */ int load(StorageFS *fs); /** * Simply sets the number of cells for the last tile. * * @param cell_num The number of cells for the last tile. * @return void */ void set_last_tile_cell_num(int64_t cell_num); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ // File backing serialized book_keeping content std::string filename_; // StorageBuffer backing the book_keeping content StorageBuffer *buffer_; // Hardcoded upload/download sizes for book-keeping segments // Each segment is individually compressed size_t upload_uncompressed_size_ = 50*1024*1024; // 50M size_t download_compressed_size_ = 10*1024*1024; // 10M /** The array schema */ const ArraySchema* array_schema_; /** The first and last coordinates of each tile. */ std::vector bounding_coords_; /** True if the fragment is dense, and false if it is sparse. */ bool dense_; /** * The (expanded) domain in which the fragment is constrained. "Expanded" * means that the domain is enlarged minimally to coincide with tile * boundaries (if there is a tile grid imposed by tile extents). Note that the * type of the domain must be the same as the type of the array coordinates. */ void* domain_; /** The name of the fragment the book-keeping belongs to. */ std::string fragment_name_; /** Number of cells in the last tile (meaningful only in the sparse case). */ int64_t last_tile_cell_num_; /** The MBRs (applicable only to the sparse case with irregular tiles). */ std::vector mbrs_; /** The mode in which the fragment was initialized. */ int mode_; /** The offsets of the next tile for each attribute. */ std::vector next_tile_offsets_; /** The offsets of the next variable tile for each attribute. */ std::vector next_tile_var_offsets_; /** * The non-empty domain in which the fragment is constrained. Note that the * type of the domain must be the same as the type of the array coordinates. */ void* non_empty_domain_; /** * The tile offsets in their corresponding attribute files. Meaningful only * when there is compression. */ std::vector > tile_offsets_; /** * The variable tile offsets in their corresponding attribute files. * Meaningful only for variable-sized tiles. */ std::vector > tile_var_offsets_; /** * The sizes of the uncompressed variable tiles. * Meaningful only when there is compression for variable tiles. */ std::vector > tile_var_sizes_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Writes the bounding coordinates to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_bounding_coords(); /** * Writes the cell number of the last tile to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_last_tile_cell_num(); /** * Writes the MBRs to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_mbrs(); /** * Writes the non-empty domain to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_non_empty_domain(); /** * Writes the tile offsets to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_tile_offsets(); /** * Writes the variable tile offsets to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_tile_var_offsets(); /** * Writes the variable tile sizes to the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int flush_tile_var_sizes(); /** * Loads the bounding coordinates from the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_bounding_coords(); /** * Loads the cell number of the last tile from the book-keeping buffer * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_last_tile_cell_num(); /** * Loads the MBRs from the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_mbrs(); /** * Loads the non-empty domain from the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_non_empty_domain(); /** * Loads the tile offsets from the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_tile_offsets(); /** * Loads the variable tile offsets from the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_tile_var_offsets(); /** * Loads the variable tile sizes from the book-keeping buffer. * @return TILEDB_BK_OK on success and TILEDB_BK_ERR on error. */ int load_tile_var_sizes(); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/fragment/fragment.h000066400000000000000000000213121453617025200244760ustar00rootroot00000000000000/** * @file fragment.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class Fragment. */ #ifndef __FRAGMENT_H__ #define __FRAGMENT_H__ #include "array.h" #include "array_schema.h" #include "book_keeping.h" #include "read_state.h" #include "write_state.h" #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_FG_OK 0 #define TILEDB_FG_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_FG_ERRMSG std::string("[TileDB::Fragment] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_fg_errmsg; class Array; class BookKeeping; class ReadState; class WriteState; /** Manages a TileDB fragment object. */ class Fragment { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param array The array the fragment belongs to. */ Fragment(const Array* array); /** Destructor. */ ~Fragment(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns the array the fragment belongs to. */ const Array* array() const; /** Returns the book_keeping data structure for this fragment. */ BookKeeping* book_keeping() const; /** Returns the number of cell per (full) tile. */ int64_t cell_num_per_tile() const; /** Returns true if the fragment is dense, and false if it is sparse. */ bool dense() const; /** Returns the fragment name. */ const std::string& fragment_name() const; /** Returns the mode of the fragment. */ int mode() const; /** Returns true if the array is in read mode. */ bool read_mode() const; /** Returns the read state of the fragment. */ ReadState* read_state() const; /** * Returns the tile size for a given attribute (TILEDB_VAR_SIZE in case * of a variable-sized attribute. */ size_t tile_size(int attribute_id) const; /** Returns true if the array is in write mode. */ bool write_mode() const; /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Finalizes the fragment, properly freeing up memory space. * * @return TILEDB_FG_OK on success and TILEDB_FG_ERR on error. */ int finalize(); /** * Initializes a fragment in write mode. * * @param fragment_name The name that will be given to the fragment. * @param mode The fragment mode. It can be one of the following: * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_UNSORTED * @param subarray The subarray the fragment is constrained on. * @return TILEDB_FG_OK on success and TILEDB_FG_ERR on error. */ int init( const std::string& fragment_name, int mode, const void* subarray); /** * Initializes a fragment in read mode. * * @param fragment_name The name that will be given to the fragment. * @param book_keeping The book-keeping of the fragment. * @param mode The fragment mode. It can be any of the read modes: * - TILEDB_ARRAY_READ * - TILEDB_ARRAY_READ_SORTED_COL * - TILEDB_ARRAY_READ_SORTED_ROW * @return TILEDB_FG_OK on success and TILEDB_FG_ERR on error. */ int init( const std::string& fragment_name, BookKeeping* book_keeping, int mode); /** Resets the read state (typically to start a new read). */ void reset_read_state(); /** * Syncs all attribute files in the fragment. * * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int sync(); /** * Syncs the currently written files associated with the input attribute * in the input array. * * @return TILEDB_AR_OK on success, and TILEDB_AR_ERR on error. */ int sync_attribute(const std::string& attribute); /** * Performs a write operation in the fragment. The cell values are provided * in a set of buffers (one per attribute specified upon initialization). * Note that there must be a one-to-one correspondance between the cell * values across the attribute buffers. * * The fragment must be initialized in one of the following write modes, * each of which having a different behaviour: * - TILEDB_ARRAY_WRITE: \n * In this mode, the cell values are provided in the buffers respecting * the cell order on the disk. It is practically an **append** operation, * where the provided cell values are simply written at the end of * their corresponding attribute files. This mode leads to the best * performance. The user may invoke this function an arbitrary number * of times, and all the writes will occur in the same fragment. * Moreover, the buffers need not be synchronized, i.e., some buffers * may have more cells than others when the function is invoked. * - TILEDB_ARRAY_WRITE_UNSORTED: \n * This mode is applicable to sparse arrays, or when writing sparse * updates to a dense array. One of the buffers holds the coordinates. * The cells in this mode are given in an arbitrary, unsorted order * (i.e., without respecting how the cells must be stored on the disk * according to the array schema definition). Each invocation of this * function internally sorts the cells and writes them to the disk on the * proper order. In addition, each invocation creates a **new** fragment. * Finally, the buffers in each invocation must be synced, i.e., they * must have the same number of cell values across all attributes. * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in Array::init() * or Array::reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second holds the variable-sized cell * values, whereas the first holds the start offsets of each cell in the * second buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @return TILEDB_FG_OK for success and TILEDB_FG_ERR for error. */ int write(const void** buffers, const size_t* buffer_sizes); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The array the fragment belongs to. */ const Array* array_; /** The fragment book-keeping. */ BookKeeping* book_keeping_; /** Indicates whether the fragment is dense or sparse. */ bool dense_; /** The fragment name. */ std::string fragment_name_; /** * The fragment mode. It must be one of the following: * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_UNSORTED * - TILEDB_ARRAY_READ */ int mode_; /** The fragment read state. */ ReadState* read_state_; /** The fragment write state. */ WriteState* write_state_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Changes the temporary fragment name into a stable one. * * @return TILEDB_FG_OK for success, and TILEDB_FG_ERR for error. */ int rename_fragment(); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/fragment/read_state.h000066400000000000000000001013631453617025200250130ustar00rootroot00000000000000/** * @file read_state.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ReadState. */ #ifndef __READ_STATE_H__ #define __READ_STATE_H__ #include "array.h" #include "book_keeping.h" #include "codec.h" #include "fragment.h" #include "storage_buffer.h" #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_RS_OK 0 #define TILEDB_RS_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_RS_ERRMSG std::string("[TileDB::ReadState] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_rs_errmsg; class Array; class Fragment; /** Stores the state necessary when reading cells from a fragment. */ class ReadState { public: /* ********************************* */ /* TYPE DEFINITIONS */ /* ********************************* */ /** A cell position pair [first, second]. */ typedef std::pair CellPosRange; /** A pair [fragment_id, tile_pos]. */ typedef std::pair FragmentInfo; /** A pair of fragment info and fragment cell position range. */ typedef std::pair FragmentCellPosRange; /** A vector of fragment cell posiiton ranges. */ typedef std::vector FragmentCellPosRanges; /** A vector of vectors of fragment cell position ranges. */ typedef std::vector FragmentCellPosRangesVec; /** * A pair of fragment info and cell range, where the cell range is defined * by two bounding coordinates. */ typedef std::pair FragmentCellRange; /** A vector of fragment cell ranges. */ typedef std::vector FragmentCellRanges; /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param fragment The fragment the read state belongs to. * @param book_keeping The book-keeping of the fragment. */ ReadState(const Fragment* fragment, BookKeeping* book_keeping); /** Destructor. */ ~ReadState(); /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Finalizes the fragment. * * @return TILEDB_WS_OK for success and TILEDB_WS_ERR for error. */ int finalize(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns *true* if the read state corresponds to a dense fragment. */ bool dense() const; /** Returns *true* if the read operation is finished for this fragment. */ bool done() const; /** * Copies the bounding coordinates of the current search tile into the input * *bounding_coords*. */ void get_bounding_coords(void* bounding_coords) const; /** * Returns *true* if the MBR of the search tile overlaps with the current * tile under investigation. Applicable only to **sparse** fragments in * **dense** arrays. NOTE: if the MBR of the search tile has not not changed * and the function is invoked again, it will return *false*. */ bool mbr_overlaps_tile() const; /** Returns *true* if the read buffers overflowed for the input attribute. */ bool overflow(int attribute_id) const; /** * True if the fragment non-empty domain fully covers the subarray area of * the current overlapping tile. */ bool subarray_area_covered() const; /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Resets the read state. Note that it does not flush any buffered tiles, so * that they can be reused later if a subsequent request happens to overlap * with them. * */ void reset(); /** * Resets the overflow flag of every attribute to *false*. * * @return void. */ void reset_overflow(); /* ********************************* */ /* MISC */ /* ********************************* */ /** * Copies the cells of the input attribute into the input buffers, as * determined by the input cell position range. * * @param attribute_id The id of the targeted attribute. * @param tile_i The tile to copy from. * @param buffer The buffer to copy into - see Array::read(). * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param cell_pos_range The cell position range to be copied. * @param remaining_skip_count The number of cells to skip before copying * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ int copy_cells( int attribute_id, int tile_i, void* buffer, size_t buffer_size, size_t& buffer_offset, const CellPosRange& cell_pos_range, size_t& remaining_skip_count); /** * Copies the cells of the input **variable-sized** attribute into the input * buffers, as determined by the input cell position range. * * @param attribute_id The id of the targeted attribute. * @param tile_i The tile to copy from. * @param buffer The offsets buffer to copy into - see Array::read(). * @param buffer_size The size (in bytes) of *buffer*. * @param buffer_offset The offset in *buffer* where the copy will start from. * @param remaining_skip_count The number of cells to skip before copying * @param buffer_var The variable-sized cell buffer to copy into - see * Array::read(). * @param buffer_var_size The size (in bytes) of *buffer_var*. * @param buffer_var_offset The offset in *buffer_var* where the copy will * start from. * @param remaining_skip_count_var The number of cells to skip before copying for the var field * @param cell_pos_range The cell position range to be copied. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ int copy_cells_var( int attribute_id, int tile_i, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var, const CellPosRange& cell_pos_range); /** * Retrieves the coordinates after the input coordinates in the search tile. * * @tparam T The coordinates type. * @param coords The target coordinates. * @param coords_after The coordinates to be retrieved. * @param coords_retrieved *true* if *coords_after* are indeed retrieved. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ template int get_coords_after( const T* coords, T* coords_after, bool& coords_retrieved); /** * Given a target coordinates set, it returns the coordinates preceding and * succeeding it in a designated tile and inside an indicated coordinate * range. * * @tparam T The coordinates type. * @param tile_i The targeted tile position. * @param target_coords The target coordinates. * @param start_coords The starting coordinates of the target cell range. * @param end_coords The ending coordinates of the target cell range. * @param left_coords The returned preceding coordinates. * @param right_coords The returned succeeding coordinates. * @param left_retrieved *true* if the preceding coordinates are retrieved. * @param right_retrieved *true* if the succeeding coordinates are retrieved. * @param target_exists *true* if the target coordinates exist in the tile. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ template int get_enclosing_coords( int tile_i, const T* target_coords, const T* start_coords, const T* end_coords, T* left_coords, T* right_coords, bool& left_retrieved, bool& right_retrieved, bool& target_exists); /** * Retrieves the cell position range corresponding to the input cell range, * for the case of **sparse** fragments. * * @tparam T The coordinates type. * @param fragment_info The (fragment id, tile position) pair corresponding * to the cell position range to be retrieved. * @param cell_range The targeted cell range. * @param fragment_cell_pos_range The result cell position range. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ template int get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const T* cell_range, FragmentCellPosRange& fragment_cell_pos_range); /** * Computes the fragment cell ranges corresponding to the current search * tile. Applicable only to **dense**. * * @tparam T The coordinates type. * @param fragment_i The fragment id. * @param fragment_cell_ranges The output fragment cell ranges. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ template int get_fragment_cell_ranges_dense( int fragment_i, FragmentCellRanges& fragment_cell_ranges); /** * Computes the fragment cell ranges corresponding to the current search * tile. Applicable only to **sparse** fragments for **dense** arrays. * * @tparam T The coordinates type. * @param fragment_i The fragment id. * @param fragment_cell_ranges The output fragment cell ranges. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ template int get_fragment_cell_ranges_sparse( int fragment_i, FragmentCellRanges& fragment_cell_ranges); /** * Computes the fragment cell ranges corresponding to the current search * tile, which are contained within the input start and end coordinates. * Applicable only to **sparse** fragments for **sparse** arrays. * * @tparam T The coordinates type. * @param fragment_i The fragment id. * @param start_coords The start coordinates of the specified range. * @param end_coords The end coordinates of the specified range. * @param fragment_cell_ranges The output fragment cell ranges. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ template int get_fragment_cell_ranges_sparse( int fragment_i, const T* start_coords, const T* end_coords, FragmentCellRanges& fragment_cell_ranges); /** * Gets the next overlapping tile from the fragment, which may overlap or not * with the tile specified by the input tile coordinates. This is applicable * only to **dense** fragments. * * @tparam T The coordinates type. * @param tile_coords The input tile coordinates. * @return void */ template void get_next_overlapping_tile_dense(const T* tile_coords); /** * Gets the next overlapping tile from the fragment. This is applicable * only to **sparse** arrays. * * @tparam T The coordinates type. * @return void */ template void get_next_overlapping_tile_sparse(); /** * Gets the next overlapping tile from the fragment, such that it overlaps or * succeeds the tile with the input tile coordinates. This is applicable * only to **sparse** fragments for **dense** arrays. * * @tparam T The coordinates type. * @param tile_coords The input tile coordinates. * @return void */ template void get_next_overlapping_tile_sparse(const T* tile_coords); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The array the fragment belongs to. */ const Array* array_; /** The array schema. */ const ArraySchema* array_schema_; /** The number of array attributes. */ int attribute_num_; /** The book-keeping of the fragment the read state belongs to. */ BookKeeping* book_keeping_; /** The size of the array coordinates. */ size_t coords_size_; /** Internal buffers associated with the attribute files */ std::vector file_buffer_; std::vector file_var_buffer_; /** Cache attribute filesizes for fragment */ std::vector file_size_; std::vector file_var_size_; /** Compression per attribute */ std::vector codec_; std::vector offsets_codec_; /** Indicates if the read operation on this fragment finished. */ bool done_; /** Keeps track of which tile is in main memory for each attribute. */ std::vector fetched_tile_; /** The fragment the read state belongs to. */ const Fragment* fragment_; /** * Last investigated tile coordinates. Applicable only to **sparse** fragments * for **dense** arrays. */ void* last_tile_coords_; /** A buffer for each attribute used by mmap for mapping a tile from disk. */ std::vector map_addr_; /** The corresponding lengths of the buffers in map_addr_. */ std::vector map_addr_lengths_; /** A buffer mapping a compressed tile from disk. */ void* map_addr_compressed_; /** The corresponding length of the map_addr_compressed_ buffer. */ size_t map_addr_compressed_length_; /** * A buffer for each attribute used by mmap for mapping a variable tile from * disk. */ std::vector map_addr_var_; /** The corresponding lengths of the buffers in map_addr_var_. */ std::vector map_addr_var_lengths_; /** * The overlap between an MBR and the current tile under investigation * in the case of **sparse** fragments in **dense** arrays. The overlap * can be one of the following: * - 0: No overlap * - 1: The query subarray fully covers the search tile * - 2: Partial overlap * - 3: Partial overlap contig */ int mbr_tile_overlap_; /** Indicates buffer overflow for each attribute. */ std::vector overflow_; /** * The type of overlap of the current search tile with the query subarray * is full or not. It can be one of the following: * - 0: No overlap * - 1: The query subarray fully covers the search tile * - 2: Partial overlap * - 3: Partial overlap contig */ int search_tile_overlap_; /** The overlap between the current search tile and the query subarray. */ void* search_tile_overlap_subarray_; /** The positions of the currently investigated tile. */ int64_t search_tile_pos_; /** * True if the fragment non-empty domain fully covers the subarray area * in the current overlapping tile. */ bool subarray_area_covered_; /** Internal buffer used in the case of compression. */ void* tile_compressed_; /** Allocated size for internal buffer used in the case of compression. */ size_t tile_compressed_allocated_size_; /** File offset for each attribute tile. */ std::vector tiles_file_offsets_; /** File offset for each variable-sized attribute tile. */ std::vector tiles_var_file_offsets_; /** * Local tile buffers, one per attribute, plus two for coordinates * (the second one is for searching). */ std::vector tiles_; /** Current offsets in tiles_ (one per attribute). */ std::vector tiles_offsets_; /** * The tile position range the search for overlapping tiles with the * subarray query will focus on. */ int64_t tile_search_range_[2]; /** Sizes of tiles_ (one per attribute). */ std::vector tiles_sizes_; /** Local variable tile buffers (one per attribute). */ std::vector tiles_var_; /** Allocated sizes for the local variable tile buffers. */ std::vector tiles_var_allocated_size_; /** Current offsets in tiles_var_ (one per attribute). */ std::vector tiles_var_offsets_; /** Sizes of tiles_var_ (one per attribute). */ std::vector tiles_var_sizes_; /** Temporary coordinates. */ void* tmp_coords_; /** Temporary offset. */ size_t tmp_offset_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Helper method to construct filename for given attribute */ std::string construct_filename(int attribute_id, bool is_var=false); /** * Resets all internal buffers associated with attribute files. */ void reset_file_buffers(); /** * Reads a segment from the file associated with the attribute, * given an offset and size. * @param attribute_id The id of the attribute. * @param is_var Boolean to specify whether the attribute is var. * @param offset The offset of the segment to be read from the file. * @param segment Pointer to preallocated segment. * @param length Length of the preallocated segment. * @return TILEDB_RS_OK on success and TILEDB_RS_ERR on error. */ int read_segment(int attribute_id, bool is_var, off_t offset, void *segment, size_t length); /** * Compares input coordinates to coordinates from the search tile. * * @param buffer The data buffer to be compared. * @param tile_offset The offset in the tile where the data comparison * starts form. * @return 1 if the compared data are equal, 0 if they are not equal and * TILEDB_RS_ERR for error. */ int CMP_COORDS_TO_SEARCH_TILE( const void* buffer, size_t tile_offset); /** * Computes the number of bytes to copy from the local tile buffers of a given * attribute in the case of variable-sized cells. It takes as input a range * of cells that should be copied ideally if the buffer sizes permit it. * Then, based on the available buffer sizes, this range is adjusted and the * final bytes to be copied for the two buffers are returned. * * @param attribute_id The id of the attribute. * @param start_cell_pos The starting position of the cell range. * @param end_cell_pos The ending position of the cell range. The function may * alter this value upon termination. * @param buffer_free_space The free space in the buffer that will store the * starting offsets of the variable-sized cells. * @param buffer_var_free_space The free space in the buffer that will store * the actual variable-sized cells. * @param bytes_to_copy The returned bytes to copy into the the buffer that * will store the starting offsets of the variable-sized cells. * @param bytes_var_to_copy The returned bytes to copy into the the buffer * that will store the actual variable-sized cells. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int compute_bytes_to_copy( int attribute_id, int64_t start_cell_pos, int64_t& end_cell_pos, size_t buffer_free_space, size_t buffer_var_free_space, size_t& bytes_to_copy, size_t& bytes_var_to_copy); /** * Computes the ranges of tile positions that need to be searched for finding * overlapping tiles with the query subarray. * * @return void */ void compute_tile_search_range(); /** * Computes the ranges of tile positions that need to be searched for finding * overlapping tiles with the query subarray. * * @tparam T The coordinates type. * @return void */ template void compute_tile_search_range(); /** * Computes the ranges of tile positions that need to be searched for finding * overlapping tiles with the query subarray. This function focuses on the * case of column- or row-major cell orders. * * @tparam T The coordinates type. * @return void */ template void compute_tile_search_range_col_or_row(); /** * Computes the ranges of tile positions that need to be searched for finding * overlapping tiles with the query subarray. This function focuses on the * case of the Hilbert cell order. * * @tparam T The coordinates type. * @return void */ template void compute_tile_search_range_hil(); /** * Decompresses a tile. * * @param attribute_id The id of the attribute the tile belongs to. * @param tile_compressed The compressed tile to be decompressed. * @param tile_compressed_size The size of the compressed tile. * @param tile The resulting decompressed tile. * @param tile_size The expected size of the decompressed tile (for checking * for errors). * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int decompress_tile( int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size, bool decompress_offsets = false); /** * Returns the cell position in the search tile that is after the * input coordinates. * * @tparam T The coordinates type. * @param coords The input coordinates. * @return The cell position in the search tile that is after the * input coordinates. */ template int64_t get_cell_pos_after(const T* coords); /** * Returns the cell position in the search tile that is at or after the * input coordinates. * * @tparam T The coordinates type. * @param coords The input coordinates. * @return The cell position in the search tile that is at or after the * input coordinates. */ template int64_t get_cell_pos_at_or_after(const T* coords); /** * Returns the cell position in the search tile that is at or before the * input coordinates. * * @tparam T The coordinates type. * @param coords The input coordinates. * @return The cell position in the search tile that is at or before the * input coordinates. */ template int64_t get_cell_pos_at_or_before(const T* coords); /** * Retrieves the pointer of the i-th coordinates in the search tile. * * @param i Indicates the i-th coordinates pointer to be retrieved. * @param coords The destination pointer. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int GET_COORDS_PTR_FROM_SEARCH_TILE( int64_t i, const void*& coords); /** * Retrieves the pointer of the i-th cell in the offset tile of a * variable-sized attribute. * * @param attribute_id The attribute id. * @param i Indicates the i-th offset pointer to be retrieved. * @param offset The destination pointer. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int GET_CELL_PTR_FROM_OFFSET_TILE( int attribute_id, int64_t i, const size_t*& offset); /** Returns *true* if the file of the input attribute is empty. */ bool is_empty_attribute(int attribute_id) const; /** * Maps a tile from the disk for an attribute into a local buffer, using * memory map (mmap). This function works with any compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int map_tile_from_file_cmp( int attribute_id, off_t offset, size_t tile_size); /** * Maps a variable-sized tile from the disk for an attribute into a local * buffer, using memory map (mmap). This function works with any compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int map_tile_from_file_var_cmp( int attribute_id, off_t offset, size_t tile_size); /** * Maps a tile from the disk for an attribute into a local buffer, using * memory map (mmap). This function focuses on the case of no compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int map_tile_from_file_cmp_none( int attribute_id, off_t offset, size_t tile_size); /** * Maps a variable-sized tile from the disk for an attribute into a local * buffer, using memory map (mmap). This function focuses on the case of * no compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int map_tile_from_file_var_cmp_none( int attribute_id, off_t offset, size_t tile_size); #ifdef HAVE_MPI /** * Reads a tile from the disk for an attribute into a local buffer, using * MPI-IO. This function focuses on the case of GZIP compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int mpi_io_read_tile_from_file_cmp( int attribute_id, off_t offset, size_t tile_size); /** * Reads a variable-sized tile from the disk for an attribute into a local * buffer, using MPI-IO. This function focuses on the case of any * compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int mpi_io_read_tile_from_file_var_cmp( int attribute_id, off_t offset, size_t tile_size); #endif /** * Prepares a tile from the disk for reading for an attribute. * * @param attribute_id The id of the attribute the tile is prepared for. * @param tile_i The tile position on the disk. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int prepare_tile_for_reading(int attribute_id, int64_t tile_i); /** * Prepares a variable-sized tile from the disk for reading for an attribute. * * @param attribute_id The id of the attribute the tile is prepared for. * @param tile_i The tile position on the disk. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int prepare_tile_for_reading_var(int attribute_id, int64_t tile_i); /** * Prepares a tile from the disk for reading for an attribute. * This function focuses on the case there is any compression. * * @param attribute_id The id of the attribute the tile is prepared for. * @param tile_i The tile position on the disk. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int prepare_tile_for_reading_cmp(int attribute_id, int64_t tile_i); /** * Prepares a tile from the disk for reading for an attribute. * This function focuses on the case there is no compression. * * @param attribute_id The id of the attribute the tile is prepared for. * @param tile_i The tile position on the disk. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int prepare_tile_for_reading_cmp_none(int attribute_id, int64_t tile_i); /** * Prepares a tile from the disk for reading for an attribute. * This function focuses on the case of variable-sized tiles with any * compression. * * @param attribute_id The id of the attribute the tile is prepared for. * @param tile_i The tile position on the disk. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int prepare_tile_for_reading_var_cmp(int attribute_id, int64_t tile_i); /** * Prepares a tile from the disk for reading for an attribute. * This function focuses on the case of variable-sized tiles with no * compression. * * @param attribute_id The id of the attribute the tile is prepared for. * @param tile_i The tile position on the disk. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int prepare_tile_for_reading_var_cmp_none(int attribute_id, int64_t tile_i); /** * Reads data from an attribute tile into an input buffer. * * @param attribute_id The attribute id. * @param buffer The destination buffer. * @param tile_offset The offset in the tile where the read starts from. * @param bytes_to_copy The number of bytes to copy from the tile into the * buffer. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int READ_FROM_TILE( int attribute_id, void* buffer, size_t tile_offset, size_t bytes_to_copy); /** * Reads data from a variable-sized attribute tile into an input buffer. * * @param attribute_id The attribute id. * @param buffer The destination buffer. * @param tile_offset The offset in the tile where the read starts from. * @param bytes_to_copy The number of bytes to copy from the tile into the * buffer. * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int READ_FROM_TILE_VAR( int attribute_id, void* buffer, size_t tile_offset, size_t bytes_to_copy); /** * Reads a tile from the disk for an attribute into a local buffer. This * function focuses on the case there is any compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int read_tile_from_file_cmp( int attribute_id, off_t offset, size_t tile_size); /** * Reads a tile from the disk for an attribute into a local buffer. This * function focuses on the case of variable-sized tiles and any compression. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @param tile_size The tile size. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int read_tile_from_file_var_cmp( int attribute_id, off_t offset, size_t tile_size); /** * Saves in the read state the file offset for an attribute tile. * This will be used in subsequent read requests. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int set_tile_file_offset( int attribute_id, off_t offset); /** * Saves in the read state the file offset for a variable-sized attribute * tile. This will be used in subsequent read requests. * * @param attribute_id The id of the attribute the read occurs for. * @param offset The offset at which the tile starts in the file. * @return TILEDB_RS_OK for success, and TILEDB_RS_ERR for error. */ int set_tile_var_file_offset( int attribute_id, off_t offset); /** * Shifts the offsets stored in the tile buffer of the input attribute, such * that the first starts from 0 and the rest are relative to the first one. * * @param attribute_id The id of the attribute the tile corresponds to. * @return void. */ void shift_var_offsets(int attribute_id); /** * Shifts the offsets stored in the input buffer such that they are relative * to the input "new_start_offset". * * @param buffer The input buffer that stores the offsets. * @param offset_num The number of offsets in the buffer. * @param new_start_offset The new starting offset, i.e., the first element * in the buffer will be equal to this value, and the rest of the offsets * will be shifted relative to this offset. * @return void. */ void shift_var_offsets( void* buffer, int64_t offset_num, size_t new_start_offset); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/fragment/write_state.h000066400000000000000000000605031453617025200252320ustar00rootroot00000000000000/** * @file write_state.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class WriteState. */ #ifndef __WRITE_STATE_H__ #define __WRITE_STATE_H__ #include "book_keeping.h" #include "codec.h" #include "fragment.h" #include "storage_buffer.h" #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_WS_OK 0 #define TILEDB_WS_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_WS_ERRMSG std::string("[TileDB::WriteState] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_ws_errmsg; class BookKeeping; class Fragment; /** Stores the state necessary when writing cells to a fragment. */ class WriteState { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param fragment The fragment the write state belongs to. * @param book_keeping The book-keeping *fragment*. */ WriteState(const Fragment* fragment, BookKeeping* book_keeping); /** Destructor. */ ~WriteState(); /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Finalizes the fragment. * * @return TILEDB_WS_OK for success and TILEDB_WS_ERR for error. */ int finalize(); /** * Syncs all attribute files in the fragment. * * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int sync(); /** * Syncs the input attribute in the fragment. * * @param attribute The attribute name. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int sync_attribute(const std::string& attribute); /** * Performs a write operation in the fragment. The cell values are provided * in a set of buffers (one per attribute specified upon the array * initialization). Note that there must be a one-to-one correspondance * between the cell values across the attribute buffers. * * The array must have been initialized in one of the following write modes, * each of which having a different behaviour: * - TILEDB_ARRAY_WRITE: \n * In this mode, the cell values are provided in the buffers respecting * the cell order on the disk. It is practically an **append** operation, * where the provided cell values are simply written at the end of * their corresponding attribute files. * - TILEDB_ARRAY_WRITE_UNSORTED: \n * This mode is applicable to sparse arrays, or when writing sparse * updates to a dense array. One of the buffers holds the coordinates. * The cells in this mode are given in an arbitrary, unsorted order * (i.e., without respecting how the cells must be stored on the disk * according to the array schema definition). * * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * Array::init() or Array::reset_attributes(). The case of variable-sized * attributes is special. Instead of providing a single buffer for such an * attribute, **two** must be provided: the second holds the * variable-sized cell values, whereas the first holds the start offsets * of each cell in the second buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @return TILEDB_WS_OK for success and TILEDB_WS_ERR for error. */ int write( const void** buffers, const size_t* buffer_sizes); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The array the fragment belongs to. */ const Array* array_; /** The array schema. */ const ArraySchema* array_schema_; /** The number of array attributes. */ int attribute_num_; /** The book-keeping structure of the fragment the write state belongs to. */ BookKeeping* book_keeping_; /** The first and last coordinates of the tile currently being populated. */ void* bounding_coords_; /** Internal buffers associated with the attribute files */ std::vector file_buffer_; std::vector file_var_buffer_; /** Compression per attribute */ std::vector codec_; std::vector offsets_codec_; /** * The current offsets of the variable-sized attributes in their * respective files, or alternatively, the current file size of each * variable-sized attribute. */ std::vector buffer_var_offsets_; /** The fragment the write state belongs to. */ const Fragment* fragment_; /** The MBR of the tile currently being populated. */ void* mbr_; /** The number of cells written in the current tile for each attribute. */ std::vector tile_cell_num_; /** Internal buffers used in the case of compression. */ std::vector tiles_; /** Offsets to the internal variable tile buffers. */ std::vector tiles_var_offsets_; /** Internal buffers used in the case of compression for variable tiles. */ std::vector tiles_var_; /** * Sizes of internal buffers used in the case of compression for variable * tiles. */ std::vector tiles_var_sizes_; /** Offsets to the internal tile buffers used in compression. */ std::vector tile_offsets_; /** The Storage Filesystem */ StorageFS *fs_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Compresses the input tile buffer, and stores it inside tile_compressed_ * member attribute. * * @param attribute_id The id of the attribute the tile belongs to. * @param tile The tile buffer to be compressed. * @param tile_size The size of the tile buffer in bytes. * @param tile_compressed The compressed tile * @param tile_compressed_size The size of the resulting compressed tile. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int compress_tile( int attribute_id, unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size, bool compress_offsets = false); /** * Compresses the current tile for the input attribute, and writes (appends) * it to its corresponding file on the disk. * * @param attribute_id The id of the attribute whose tile is compressed and * written. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int compress_and_write_tile(int attribute_id); /** * Compresses the current variable-sized tile for the input attribute, and * writes (appends) it to its corresponding file on the disk. * * @param attribute_id The id of the attribute whose tile is compressed and * written. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int compress_and_write_tile_var(int attribute_id); /** * Expands the current MBR with the input coordinates. * * @tparam T The type of the MBR and the input coordinates. * @param coords The input coordinates. * @return void */ template void expand_mbr(const T* coords); /** * Shifts the offsets of the variable-sized cells recorded in the input * buffer, so that they correspond to the actual offsets in the corresponding * attribute file. * * @param attribute_id The id of the attribute whose variable cell offsets are * shifted. * @param buffer_var_size The total size of the variable-sized cells written * during this write operation. * @param buffer Holds the offsets of the variable-sized cells for this write * operation. * @param buffer_size The size (in bytes) of *buffer*. * @param shifted_buffer Will hold the new shifted offsets. * @return void */ void shift_var_offsets( int attribute_id, size_t buffer_var_size, const void* buffer, size_t buffer_size, void* shifted_buffer); /** * Sorts the input cell coordinates according to the order specified in the * array schema. This is not done in place; the sorted positions are stored * in a separate vector. * * @param buffer The buffer holding the cell coordinates. * @param buffer_size The size (in bytes) of *buffer*. * @param cell_pos The sorted cell positions. * @return void */ void sort_cell_pos( const void* buffer, size_t buffer_size, std::vector& cell_pos) const; /** * Sorts the input cell coordinates according to the order specified in the * array schema. This is not done in place; the sorted positions are stored * in a separate vector. * * @tparam T The type of coordinates stored in *buffer*. * @param buffer The buffer holding the cell coordinates. * @param buffer_size The size (in bytes) of *buffer*. * @param cell_pos The sorted cell positions. * @return void */ template void sort_cell_pos( const void* buffer, size_t buffer_size, std::vector& cell_pos) const; /** * Updates the book-keeping structures as tiles are written. Specifically, it * updates the MBR and bounding coordinates of each tile. * * @param buffer The buffer storing the cell coordinates. * @param buffer_size The size (in bytes) of *buffer*. * @return void */ void update_book_keeping(const void* buffer, size_t buffer_size); /** * Updates the book-keeping structures as tiles are written. Specifically, it * updates the MBR and bounding coordinates of each tile. * * @tparam T The coordinates type. * @param buffer The buffer storing the cell coordinates. * @param buffer_size The size (in bytes) of *buffer*. * @return void */ template void update_book_keeping(const void* buffer, size_t buffer_size); std::string construct_filename(int attribute_id, bool is_var); /** * Set up memory buffers to cache bytes to be ultimately written out * to the files */ void init_file_buffers(); /** * Persist the bytes in the memory buffers to the associated files * * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_file_buffers(); /** * Writes a segment from the file associated with the attribute. * @param attribute_id The id of the attribute. * @param is_var Boolean to specify whether the attribute is var. * @param segment Pointer to segment to be written out. * @param length Length of the segment to be written out. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_segment(int attribute_id, bool is_var, const void *segment, size_t length); /** * Takes the appropriate actions for writing the very last tile of this write * operation, such as updating the book-keeping structures, and compressing * and writing the last tile on the disk. This is done for every attribute. * * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_last_tile(); /** * Performs the write operation for the case of a dense fragment. * * @param buffers See write(). * @param buffer_sizes See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense( const void** buffers, const size_t* buffer_sizes); /** * Performs the write operation for the case of a dense fragment, focusing * on a single fixed-sized attribute. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense_attr( int attribute_id, const void* buffer, size_t buffer_size); /** * Performs the write operation for the case of a dense fragment, focusing * on a single fixed-sized attribute and the case of no compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense_attr_cmp_none( int attribute_id, const void* buffer, size_t buffer_size); /** * Performs the write operation for the case of a dense fragment, focusing * on a single fixed-sized attribute and the case of any compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense_attr_cmp( int attribute_id, const void* buffer, size_t buffer_size); /** * Performs the write operation for the case of a dense fragment, focusing * on a single variable-sized attribute. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense_attr_var( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size); /** * Performs the write operation for the case of a dense fragment, focusing * on a single variable-sized attribute and the case of no compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense_attr_var_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size); /** * Performs the write operation for the case of a dense fragment, focusing * on a single variable-sized attribute and the case of any compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_dense_attr_var_cmp( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size); /** * Performs the write operation for the case of a sparse fragment. * * @param buffers See write(). * @param buffer_sizes See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse( const void** buffers, const size_t* buffer_sizes); /** * Performs the write operation for the case of a sparse fragment, focusing * on a single fixed-sized attribute. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_attr( int attribute_id, const void* buffer, size_t buffer_size); /** * Performs the write operation for the case of a sparse fragment, focusing * on a single fixed-sized attribute and the case of no compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_attr_cmp_none( int attribute_id, const void* buffer, size_t buffer_size); /** * Performs the write operation for the case of a sparse fragment, focusing * on a single fixed-sized attribute and the case of any compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_attr_cmp( int attribute_id, const void* buffer, size_t buffer_size); /** * Performs the write operation for the case of a sparse fragment, focusing * on a single variable-sized attribute. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_attr_var( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size); /** * Performs the write operation for the case of a sparse fragment, focusing * on a single variable-sized attribute and the case of no compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_attr_var_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size); /** * Performs the write operation for the case of a sparse fragment, focusing * on a single variable-sized attribute and the case of any compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_attr_var_cmp( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted. * * @param buffers See write(). * @param buffer_sizes See write(). * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted( const void** buffers, const size_t* buffer_sizes); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted, focusing on a single fixed-sized attribute. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @param cell_pos The sorted positions of the cells. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted_attr( int attribute_id, const void* buffer, size_t buffer_size, const std::vector& cell_pos); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted, focusing on a single fixed-sized attribute and * the case of no compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @param cell_pos The sorted positions of the cells. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted_attr_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const std::vector& cell_pos); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted, focusing on a single fixed-sized attribute and * the case of any compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write(). * @param buffer_size See write(). * @param cell_pos The sorted positions of the cells. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted_attr_cmp( int attribute_id, const void* buffer, size_t buffer_size, const std::vector& cell_pos); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted, focusing on a single variable-sized attribute. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @param cell_pos The sorted positions of the cells. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted_attr_var( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size, const std::vector& cell_pos); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted, focusing on a single variable-sized attribute and * the case of no compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @param cell_pos The sorted positions of the cells. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted_attr_var_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size, const std::vector& cell_pos); /** * Performs the write operation for the case of a sparse fragment when the * coordinates are unsorted, focusing on a single variable-sized attribute and * the case of any compression. * * @param attribute_id The id of the attribute this operation focuses on. * @param buffer See write() - start offsets in *buffer_var*. * @param buffer_size See in write(). * @param buffer_var See write() - actual variable-sized values. * @param buffer_var_size See write(). * @param cell_pos The sorted positions of the cells. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int write_sparse_unsorted_attr_var_cmp( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size, const std::vector& cell_pos); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/metadata/000077500000000000000000000000001453617025200225005ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/metadata/metadata.h000066400000000000000000000253141453617025200244360ustar00rootroot00000000000000/** * @file metadata.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class Metadata. */ #ifndef __METADATA_H__ #define __METADATA_H__ #include "array.h" #include "storage_manager_config.h" /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_MT_OK 0 #define TILEDB_MT_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_MT_ERRMSG std::string("[TileDB::Metadata] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_mt_errmsg; /** Manages a TileDB metadata object. */ class Metadata { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ Metadata(); /** Destructor. */ ~Metadata(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns the array that implements the metadata. */ Array* array() const; /** Returns the array schema. */ const ArraySchema* array_schema() const; /** * Checks if a read operation for a particular attribute resulted in a * buffer overflow. * * @param attribute_id The id of the attribute for which the overflow is * checked. This id corresponds to the position of the attribute name * placed in the *attributes* input of init(), or reset_attributes(). If * *attributes* was NULL in the above functions, then the attribute id * corresponds to the order in which the attributes were defined in the * array schema upon the array creation. Note that, in that case, the * extra key attribute corresponds to the last extra attribute, i.e., its * id is *attribute_num*. * @return *true* for overflow and *false* otherwise. */ bool overflow(int attribute_id) const; /** * Performs a read operation in a metadata object, which must be initialized * with mode TILEDB_METADATA_READ. The read is performed on a single key. * * @param key This is the query key, which must be a string. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in init() or * reset_attributes(). The case of variable-sized attributes is special. * Instead of providing a single buffer for such an attribute, **two** * must be provided: the second will hold the variable-sized values, * whereas the first holds the start offsets of each value in the second * buffer. * @param buffer_sizes The sizes (in bytes) allocated by the user for the * input buffers (there is a one-to-one correspondence). The function will * attempt to write value corresponding to the key. If a buffer cannot * hold the result, the function will still succeed, turning on an * overflow flag which can be checked with function overflow(). * @return TILEDB_MT_OK for success and TILEDB_MT_ERR for error. */ int read(const char* key, void** buffers, size_t* buffer_sizes); /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Consolidates all fragments into a new single one, on a per-attribute basis. * Returns the new fragment (which has to be finalized outside this functions), * along with the names of the old (consolidated) fragments (which also have * to be deleted outside this function). * * @param new_fragment The new fragment to be returned. * @param old_fragment_names The names of the old fragments to be returned. * @return TILEDB_AR_OK for success and TILEDB_AR_ERR for error. */ int consolidate( Fragment*& new_fragment, std::vector& old_fragment_names); /** * Finalizes the metadata, properly freeing up the memory space. * * @return TILEDB_MT_OK on success, and TILEDB_MT_ERR on error. */ int finalize(); /** * Initializes a TileDB metadata object. * * @param array_schema This essentially encapsulates the metadata schema. * @param fragment_names The names of the fragments of the array. * @param book_keeping The book-keeping structures of the fragments * of the array. * @param mode The mode of the metadata. It must be one of the following: * - TILEDB_METADATA_WRITE * - TILEDB_METADATA_READ * @param attributes A subset of the metadata attributes the read/write will * be constrained on. A NULL value indicates **all** attributes (including * the key as an extra attribute in the end). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param config Congiguration parameters. * @return TILEDB_MT_OK on success, and TILEDB_MT_ERR on error. */ int init( const ArraySchema* array_schema, const std::vector& fragment_names, const std::vector& book_keeping, int mode, const char** attributes, int attribute_num, const StorageManagerConfig* config); /** * Resets the attributes used upon initialization of the metadata. * * @param attributes The new attributes to focus on. If it is NULL, then * all the attributes are used (including the key as an extra attribute * in the end). * @param attribute_num The number of the attributes. If *attributes* is NULL, * then this should be 0. * @return TILEDB_MT_OK on success, and TILEDB_MT_ERR on error. */ int reset_attributes(const char** attributes, int attribute_num); /** * Performs a write operation in metadata object. The values are provided * in a set of buffers (one per attribute specified upon initialization). * Note that there must be a one-to-one correspondance between the * values across the attribute buffers. * * The metadata must be initialized with mode TILEDB_METADATA_WRITE. * * @param keys The buffer holding the metadata keys. These keys must be * strings, serialized one after the other in the *keys* buffer. * @param keys_size The size (in bytes) of buffer *keys*. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second holds the variable-sized values, * whereas the first holds the start offsets of each value in the second * buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @return TILEDB_MT_OK for success and TILEDB_MT_ERR for error. */ int write( const char* keys, size_t keys_size, const void** buffers, const size_t* buffer_sizes); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The underlying array that implements the metadata. */ Array* array_; /** * The metadata mode. It must be one of the following: * - TILEDB_METADATA_WRITE * - TILEDB_METADATA_READ */ int mode_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Computes the coordinates for each key (through the MD5 hash function), * which will be used when storing the metadata to the underlying array. * * @param keys The buffer holding the metadata keys. These keys must be * strings, serialized one after the other in the *keys* buffer. * @param keys_size The size (in bytes) of buffer *keys*. * @param coords A buffer holding the computed coordinates for *keys*. * @param coords_size The size (in bytes) of the input *coords* buffer. * @return void */ void compute_array_coords( const char* keys, size_t keys_size, void*& coords, size_t& coords_size) const; /** * Prepares the buffers that will be passed to the underlying array when * writing metadata. * * @param coords A buffer holding the computed coordinates for the keys to be * written. * @param coords_size The size (in bytes) of the input *coords* buffer. * @param buffers An array of buffers, one for each attribute. These must be * provided in the same order as the attributes specified in * init() or reset_attributes(). The case of variable-sized attributes is * special. Instead of providing a single buffer for such an attribute, * **two** must be provided: the second holds the variable-sized values, * whereas the first holds the start offsets of each value in the second * buffer. * @param buffer_sizes The sizes (in bytes) of the input buffers (there is * a one-to-one correspondence). * @param array_buffers These are the produced buffers that will be passed * in the write() function. * @param array_buffer_sizes The sizes (in bytes) of the corresponding buffers * in the *buffers* parameter. * @return void */ void prepare_array_buffers( const void* coords, size_t coords_size, const void** buffers, const size_t* buffer_sizes, const void**& array_buffers, size_t*& array_buffer_sizes) const; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/metadata/metadata_iterator.h000066400000000000000000000122071453617025200263440ustar00rootroot00000000000000/** * @file metadata_iterator.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class MetadataIterator. */ #ifndef __METADATA_ITERATOR_H__ #define __METADATA_ITERATOR_H__ #include "array_iterator.h" #include "metadata.h" /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_MIT_OK 0 #define TILEDB_MIT_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_MIT_ERRMSG std::string("[TileDB::MetadataIterator] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_mit_errmsg; /** Enables iteration (read) over metadata values. */ class MetadataIterator { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ MetadataIterator(); /** Destructor. */ ~MetadataIterator(); /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Return the metadata name. */ const std::string& metadata_name() const; /** * Checks if the the iterator has reached its end. * * @return *true* if the iterator has reached its end and *false* otherwise. */ bool end() const; /** * Retrieves the current value for a particular attribute. * * @param attribute_id The id of the attribute for which the value * is retrieved. This id corresponds to the position of the attribute name * placed in the *attributes* input of init(). If *attributes* was NULL in * the above function, then the attribute id corresponds to the order in * which the attributes were defined in the array schema upon the array * creation. Note that, in that case, the extra key attribute corresponds * to the last extra attribute, i.e., its id is *attribute_num*. * @param value The value to be retrieved. Note that its type is the * same as that defined in the metadata schema. * @param value_size The size (in bytes) of the retrieved value. * @return TILEDB_MIT_OK on success, and TILEDB_MIT_ERR on error. */ int get_value(int attribute_id, const void** value, size_t* value_size) const; // MUTATORS /** * Finalizes the iterator, properly freeing the allocating memory space. * * @return TILEDB_MIT_OK on success, and TILEDB_MIT_ERR on error. */ int finalize(); /** * Initializes a metadata iterator on an already initialized metadata object. * * @param metadata The metadata the iterator is initialized for. * @param buffers This is an array of buffers similar to Metadata::read. * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read values. The iterator will * read from the disk the values in batches, by fitting as many * values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding sizes (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many values as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @return TILEDB_MIT_OK on success, and TILEDB_MIT_ERR on error. */ int init( Metadata* metadata, void** buffers, size_t* buffer_sizes); /** * Advances the iterator by one position. * * @return TILEDB_MIT_OK on success, and TILEDB_MIT_ERR on error. */ int next(); private: // PRIVATE ATTRIBUTES /** The array iterator that implements the metadata iterator. */ ArrayIterator* array_it_; /** The metadata this iterator belongs to. */ Metadata* metadata_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/metadata/metadata_schema_c.h000066400000000000000000000055621453617025200262630ustar00rootroot00000000000000/** * @file metadata_schema_c.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * A C-style struct that specifies the metadata schema. */ #ifndef __METADATA_SCHEMA_C_H__ #define __METADATA_SCHEMA_C_H__ #include /** Specifies the metadata schema. */ typedef struct MetadataSchemaC { /** * The TileDB metadata workspace. */ char* metadata_workspace_; /** * The metadata name. It is a directory, whose parent must be a TileDB * workspace, group, or array. */ char* metadata_name_; /** The attribute names. */ char** attributes_; /** The number of attributes. */ int attribute_num_; /** * The tile capacity. If it is <=0, TileDB will use its default. */ int64_t capacity_; /** * Specifies the number of values per attribute for a cell. If it is NULL, * then each attribute has a single value per cell. If for some attribute * the number of values is variable (e.g., in the case off strings), then * TILEDB_VAR_NUM must be used. */ int* cell_val_num_; /** * The compression type for each attribute (plus one extra at the end for the * key. It can be one of the following: * - TILEDB_NO_COMPRESSION * - TILEDB_GZIP * - TILEDB_ZSTD * - TILEDB_LZ4 * - TILEDB_BLOSC * - TILEDB_BLOSC_LZ4 * - TILEDB_BLOSC_LZ4HC * - TILEDB_BLOSC_SNAPPY * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE */ int* compression_; int* compression_level_; /** * The attribute types. * The attribute type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR. */ int* types_; } MetadataSchemaC; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/000077500000000000000000000000001453617025200216535ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/comparators.h000066400000000000000000000142611453617025200243620ustar00rootroot00000000000000/** * @file comparators.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Defines custom comparators to be used in cell position sorting in the case * of sparse arrays. */ #ifndef __COMPARATORS_H__ #define __COMPARATORS_H__ #define __STDC_FORMAT_MACROS #include #include /** * Wrapper of comparison function for sorting cells; first by the smallest id, * and then by column-major order of coordinates. */ template class SmallerIdCol { public: /** * Constructor. * * @param buffer The buffer containing the cells to be sorted. * @param dim_num The number of dimensions of the cells. * @param ids The ids of the cells in the buffer. */ SmallerIdCol(const T* buffer, int dim_num, const std::vector& ids) : buffer_(buffer), dim_num_(dim_num), ids_(ids) { } /** * Comparison operator. * * @param a The first cell position in the cell buffer. * @param b The second cell position in the cell buffer. */ bool operator () (int64_t a, int64_t b) { if(ids_[a] < ids_[b]) return true; if(ids_[a] > ids_[b]) return false; // a.id_ == b.id_ --> check coordinates const T* coords_a = &buffer_[a * dim_num_]; const T* coords_b = &buffer_[b * dim_num_]; for(int i=dim_num_-1; i>=0; --i) if(coords_a[i] < coords_b[i]) return true; else if(coords_a[i] > coords_b[i]) return false; // else coords_a[i] == coords_b[i] --> continue return false; } private: /** Cell buffer. */ const T* buffer_; /** Number of dimensions. */ int dim_num_; /** The cell ids. */ const std::vector& ids_; }; /** * Wrapper of comparison function for sorting cells; first by the smallest id, * and then by row-major order of coordinates. */ template class SmallerIdRow { public: /** * Constructor. * * @param buffer The buffer containing the cells to be sorted. * @param dim_num The number of dimensions of the cells. * @param ids The ids of the cells in the buffer. */ SmallerIdRow(const T* buffer, int dim_num, const std::vector& ids) : buffer_(buffer), dim_num_(dim_num), ids_(ids) { } /** * Comparison operator. * * @param a The first cell position in the cell buffer. * @param b The second cell position in the cell buffer. */ bool operator () (int64_t a, int64_t b) { if(ids_[a] < ids_[b]) return true; if(ids_[a] > ids_[b]) return false; // a.id_ == b.id_ --> check coordinates const T* coords_a = &buffer_[a * dim_num_]; const T* coords_b = &buffer_[b * dim_num_]; for(int i=0; i coords_b[i]) return false; // else coords_a[i] == coords_b[i] --> continue } return false; } private: /** Cell buffer. */ const T* buffer_; /** Number of dimensions. */ int dim_num_; /** The cell ids. */ const std::vector& ids_; }; /** Wrapper of comparison function for sorting cells on column-major order. */ template class SmallerCol { public: /** * Constructor. * * @param buffer The buffer containing the cells to be sorted. * @param dim_num The number of dimensions of the cells. */ SmallerCol(const T* buffer, int dim_num) : buffer_(buffer), dim_num_(dim_num) { } /** * Comparison operator. * * @param a The first cell position in the cell buffer. * @param b The second cell position in the cell buffer. */ bool operator () (int64_t a, int64_t b) { const T* coords_a = &buffer_[a * dim_num_]; const T* coords_b = &buffer_[b * dim_num_]; for(int i=dim_num_-1; i>=0; --i) if(coords_a[i] < coords_b[i]) return true; else if(coords_a[i] > coords_b[i]) return false; // else coords_a[i] == coords_b[i] --> continue return false; } private: /** Cell buffer. */ const T* buffer_; /** Number of dimensions. */ int dim_num_; }; /** Wrapper of comparison function for sorting cells on row-major order. */ template class SmallerRow { public: /** * Constructor. * * @param buffer The buffer containing the cells to be sorted. * @param dim_num The number of dimensions of the cells. */ SmallerRow(const T* buffer, int dim_num) : buffer_(buffer), dim_num_(dim_num) { } /** * Comparison operator. * * @param a The first cell position in the cell buffer. * @param b The second cell position in the cell buffer. */ bool operator () (int64_t a, int64_t b) { const T* coords_a = &buffer_[a * dim_num_]; const T* coords_b = &buffer_[b * dim_num_]; for(int i=0; i coords_b[i]) return false; // else coords_a[i] == coords_b[i] --> continue return false; } private: /** Cell buffer. */ const T* buffer_; /** Number of dimensions. */ int dim_num_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/error.h000066400000000000000000000102551453617025200231600ustar00rootroot00000000000000/** * @file error.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018,2020-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION Common Error Handling * */ #ifndef TILEDB_ERROR_H #define TILEDB_ERROR_H #include #include #include #include #include #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << x << std::endl #else # define PRINT_ERROR(x) do { } while(0) #endif #define SYSTEM_ERROR(PREFIX, MSG, PATH, TILEDB_MSG) \ do { \ std::string errmsg = PREFIX + "(" + __func__ + ") " + MSG; \ std::string errpath = PATH; \ if (errpath.length() > 0) { \ errmsg += " path=" + errpath; \ } \ if (errno > 0) { \ errmsg += " errno=" + std::to_string(errno) + "(" + std::string(std::strerror(errno)) + ")"; \ } \ PRINT_ERROR(errmsg); \ TILEDB_MSG = errmsg; \ } while (false) // To be used when errno has to be ignored as with awssdk api. Most other times // SYSTEM_ERROR is more appropriate. #define PATH_ERROR(PREFIX, MSG, PATH, TILEDB_MSG) \ do { \ std::string errmsg = PREFIX + "(" + __func__ + ") " + MSG; \ std::string errpath = PATH; \ if (errpath.length() > 0) { \ errmsg += " path=" + errpath; \ } \ PRINT_ERROR(errmsg); \ TILEDB_MSG = errmsg; \ } while (false) #define TILEDB_ERROR_WITH_ERRNO(PREFIX, MSG, TILEDB_MSG) \ do { \ std::string errmsg = PREFIX + "(" + __func__ + ") " + MSG; \ if (errno > 0) { \ errmsg += " errno=" + std::to_string(errno) + "(" + std::string(std::strerror(errno)) + ")"; \ } \ PRINT_ERROR(errmsg); \ TILEDB_MSG = errmsg; \ } while (false) #define TILEDB_ERROR(PREFIX, MSG, TILEDB_MSG) \ do { \ std::string errmsg = PREFIX + "(" + __func__ + ") " + MSG; \ PRINT_ERROR(errmsg); \ TILEDB_MSG = errmsg; \ } while (false) void reset_errno(); #endif /* TILEDB_ERROR_H */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/hilbert_curve.h000066400000000000000000000214541453617025200246670ustar00rootroot00000000000000/** * @file hilbert_curve.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the HilbertCurve class. */ #ifndef __HILBERT_CURVE_H__ #define __HILBERT_CURVE_H__ #include #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /** * Maximum number of dimensions for defining the Hilbert curve. Although the * Hilbert curve can be defined over arbitrary dimensionality, we limit the * number of dimensions because they affect the number of bits used to * represent a Hilbert value; in our class, a Hilbert value is a int64_t * number and, thus, it cannot exceed 64 bits. */ #define HC_MAX_DIM 16 /** * The Hilbert curve fills a multi-dimensional space in a particular manner * with a 1D line. The typical operations of this class is converting a * multi-dimensional tuple of coordinates into a 1D Hilbert value, and * vice versa. * * For the 2D case, the Hilbert curve looks as follows: * \verbatim | 15 | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ | | | | | | | | | | | | | | | | | | @ @---@ @ @ @---@ @ @ @---@ @ @ @---@ @ | | | | | | | | | | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ | | | | | | | | | | @---@ @---@---@---@ @---@ @---@ @---@---@---@ @---@ | | | | | | @ @---@---@ @---@---@ @ @ @---@---@ @---@---@ @ | | | | | | | | | | | | | Dim[1]| @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ | | | | | | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ | | | | | | | | | | | | | | @ @---@---@ @---@---@ @---@ @---@---@ @---@---@ @ | | | | @---@ @---@---@ @---@---@ @---@---@ @---@---@ @---@ | | | | | | | | | | | | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ | | | | | | | | @ @---@ @ @---@ @---@ @---@ @---@ @ @---@ @ | | | | | | | | | | | | | | | | @---@ @---@ @ @---@---@ @---@---@ @ @---@ @---@ | | | 3 | 5---6 9---@ @ @---@---@ @---@---@ @ @---@ @---@ | | | | | | | | | | | | | | | 2 | 4 7---8 @ @---@ @---@ @---@ @---@ @ @---@ @ | | | | | | | 1 | 3---2 @---@ @---@ @---@ @---@ @---@ @---@ @---@ | | | | | | | | | | | 0 | 0---1 @---@---@ @---@---@ @---@---@ @---@---@ @--255 | ------------------------------------------------------------------- 0 1 2 3 Dim[0] 15 \endverbatim * * The Hilbert value of (2,3) is 9, whereas the coordinates corresponding to * Hilbert value 2 are (1,1). * * The class utilizes two functions from John Skilling's work published in: * John Skilling, "Programming the Hilbert Curve". In AIP, 2004 */ class HilbertCurve { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param bits Number of bits used for coordinate values across each dimension * @param dim_num Number of dimensions */ HilbertCurve(int bits, int dim_num); /** Destructor. */ ~HilbertCurve(); /* ********************************* */ /* MAIN FUNCTIONS */ /* ********************************* */ /** * Converts a set of coordinates to a Hilbert value. * * @param coords The coordinates to be converted. * @param hilbert The output Hilbert value. * @return void */ void coords_to_hilbert(const int* coords, int64_t& hilbert); /** * Converts a Hilbert value into a set of coordinates. * * @param hilbert The Hilbert value to be converted. * @param coords The output coordinates. * @return void */ void hilbert_to_coords(int64_t hilbert, int* coords); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** Number of bits for representing a coordinate per dimension. */ int bits_; /** Number of dimensions. */ int dim_num_; /** Temporary buffer. */ int temp_[HC_MAX_DIM]; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Identical to John Skilling's work. It converts the input coordinates * to what is called the transpose of the Hilbert value. This is done * in place. * * **Example** * * Let b=5 and n=3. Let the 15-bit Hilbert value of the input coordinates * be A B C D E a b c d e 1 2 3 4 5. The function places this number into * parameter X as follows: * \verbatim X[0] = A D b e 3 X[1]| X[1] = B E c 1 4 <-------> | /X[2] X[2] = C a d 2 5 axes | / high low |/______ X[0] \endverbatim * * The X value after the function terminates is called the transpose form * of the Hilbert value. * * @param X Input coordinates, and output transpose (the conversion is * done in place). * @param b Number of bits for representing a coordinate per dimension * (it should be equal to HilbertCurve::bits_). * @param n Number of dimensions (it should be equal to * HilbertCurve::dim_num_). * @return void */ void AxestoTranspose(int* X, int b, int n); /** * Identical to John Skilling's work. It converts the transpose of a * Hilbert value into the corresponding coordinates. This is done in place. * * **Example** * * Let b=5 and n=3. Let the 15-bit Hilbert value of the output coordinates * be A B C D E a b c d e 1 2 3 4 5. The function takes as input the tranpose * form of the Hilbert value, which is stored in X as follows: * \verbatim X[0] = A D b e 3 X[1]| X[1] = B E c 1 4 <-------> | /X[2] X[2] = C a d 2 5 axes | / high low |/______ X[0] \endverbatim * * The X value after the function terminates will contain the coordinates * corresponding to the Hilbert value. * * @param X Input transpose, and output coordinates (the conversion is * done in place). * @param b Number of bits for representing a coordinate per dimension * (it should be equal to HilbertCurve::bits_). * @param n Number of dimensions (it should be equal to * HilbertCurve::dim_num_). * @return void */ void TransposetoAxes(int* X, int b, int n); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/mem_utils.h000066400000000000000000000026351453617025200240300ustar00rootroot00000000000000/** * @file utils.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file specifies the memory utility functions */ #pragma once void print_memory_stats(const std::string& msg); void trim_memory(); genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/progress_bar.h000066400000000000000000000065341453617025200245240ustar00rootroot00000000000000/** * @file progress_bar.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class ProgressBar. */ #ifndef __PROGRESS_BAR_H__ #define __PROGRESS_BAR_H__ #include /** The default complete amount of the bar. */ #define PB_COMPLETE 1.0 /** The default filler character of the bar. */ #define PB_FILLER '=' /** The default maximum length of the bar. */ #define PB_MAX_LENGTH 30 /** The increase in the incomplete/complete ratio before the next print. */ #define PB_RATIO_STEP 0.01 /** Implements a simple progress bar printed in standard output. */ class ProgressBar { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** * Constructor. * * @param complete The amount at which the bar must reach its maximum length. * @param max_length The visual length of the bar. * @param filler The character that fills the bar. */ ProgressBar( double complete = PB_COMPLETE, int max_length = PB_MAX_LENGTH, char filler = PB_FILLER); /** Destructor. */ ~ProgressBar(); /* ********************************* */ /* METHODS */ /* ********************************* */ /** * "Loads" the progress bar with the input amount, which may trigger drawing * the current progress on the standard output. */ void load(double amount); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The amount at which the bar reaches its maximum length. */ double complete_; /** The character that fills the bar. */ char filler_; /** The current amount accummulated towards completion. */ double incomplete_; /** The incomplete/complete ratio upon the last print. */ double last_ratio_; /** The bar current length. */ int length_; /** The bar maximum length. */ int max_length_; /** The current ratio incomplete/complete. */ double ratio_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** Prints the bar with its current status. */ void print(); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/tiledb_openssl_shim.h000066400000000000000000000203461453617025200260570ustar00rootroot00000000000000/** * @file tiledb_openssl_shim.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2023 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file provides a shim to both OpenSSL 1.x and 3.x versions. Clients can * choose to use the 3.x api if available using "OpenSSL_version_num() < 0x30000000L" for 1.x * and "OpenSSL_version_num() >= 0x30000000L" for 3.x. */ #pragma once #ifdef __cplusplus extern "C" { #endif // General unsigned long OpenSSL_version_num(void); // OpenSSL 1.x prototypes - see hmac.h typedef struct hmac_ctx_st HMAC_CTX; typedef struct evp_md_st EVP_MD; typedef struct engine_st ENGINE; typedef struct evp_md_ctx_st EVP_MD_CTX; typedef struct SHA256state_st SHA256_CTX; HMAC_CTX* __attribute__((weak)) HMAC_CTX_new(void); int __attribute__((weak)) HMAC_CTX_reset(HMAC_CTX*); int __attribute__((weak)) HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int key_len, const EVP_MD *md, ENGINE *impl); int __attribute__((weak)) HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, size_t len); int __attribute__((weak)) HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len); void __attribute__((weak)) HMAC_CTX_free(HMAC_CTX *ctx); // See md5.h #define MD5_LONG unsigned int #define MD5_CBLOCK 64 #define MD5_LBLOCK (MD5_CBLOCK / 4) typedef struct MD5state_st { MD5_LONG A, B, C, D; MD5_LONG Nl, Nh; MD5_LONG data[MD5_LBLOCK]; unsigned int num; } MD5_CTX; #define MD5_DIGEST_LENGTH 16 #define SHA256_DIGEST_LENGTH 32 int __attribute__((weak)) MD5_Init(MD5_CTX* c); int __attribute__((weak)) MD5_Update(MD5_CTX* c, const void* data, size_t); int __attribute__((weak)) MD5_Final(unsigned char*, MD5_CTX* c); unsigned char __attribute__((weak)) *MD5(const unsigned char *d, size_t n, unsigned char *md); // See sha.h #define SHA_LONG unsigned int #define SHA_LBLOCK 16 typedef struct SHA256state_st { SHA_LONG h[8]; SHA_LONG Nl, Nh; SHA_LONG data[SHA_LBLOCK]; unsigned int num, md_len; } SHA256_CTX; int __attribute__((weak)) SHA256_Init(SHA256_CTX *c); int __attribute__((weak)) SHA256_Update(SHA256_CTX *c, const void*, size_t); int __attribute__((weak)) SHA256_Final(unsigned char *md, SHA256_CTX *c); // OpenSSL 3 prototypes - see evp.h #define OSSL_ALG_PARAM_DIGEST "digest" /* utf8_string */ #define OSSL_MAC_PARAM_DIGEST OSSL_ALG_PARAM_DIGEST /* utf8 string */ #define OPENSSL_INIT_LOAD_CRYPTO_STRINGS 0x00000002L #define OPENSSL_INIT_ADD_ALL_CIPHERS 0x00000004L #define OPENSSL_INIT_ADD_ALL_DIGESTS 0x00000008L #define EVP_MD_CTX_FLAG_NON_FIPS_ALLOW 0x0008 #define EVP_CTRL_AEAD_GET_TAG 0x10 #define EVP_CTRL_AEAD_SET_TAG 0x11 #define EVP_CTRL_GCM_GET_TAG EVP_CTRL_AEAD_GET_TAG #define EVP_CTRL_GCM_SET_TAG EVP_CTRL_AEAD_SET_TAG #define EVP_MD_CTX_create() EVP_MD_CTX_new() #define EVP_MD_CTX_destroy(ctx) EVP_MD_CTX_free((ctx)) //#define EVP_MD_size EVP_MD_get_size #define EVP_CIPHER_CTX_init(c) EVP_CIPHER_CTX_reset(c) #define EVP_CIPHER_CTX_cleanup(c) EVP_CIPHER_CTX_reset(c) #define OPENSSL_add_all_algorithms_noconf() \ OPENSSL_init_crypto(OPENSSL_INIT_ADD_ALL_CIPHERS \ | OPENSSL_INIT_ADD_ALL_DIGESTS, NULL) // See params.h and types.h struct ossl_param_st { const char* key; /* the name of the parameter */ unsigned int data_type; /* declare what kind of content is in buffer */ void* data; /* value being passed in or out */ size_t data_size; /* data size */ size_t return_size; /* returned content size */ }; typedef struct evp_mac_st EVP_MAC; typedef struct evp_mac_ctx_st EVP_MAC_CTX; typedef struct ossl_lib_ctx_st OSSL_LIB_CTX; typedef struct evp_cipher_st EVP_CIPHER; typedef struct evp_cipher_ctx_st EVP_CIPHER_CTX; typedef struct ossl_param_st OSSL_PARAM; typedef struct ossl_init_settings_st OPENSSL_INIT_SETTINGS; EVP_MAC* __attribute__((weak)) EVP_MAC_fetch(OSSL_LIB_CTX*, const char*, const char*); EVP_MAC_CTX* __attribute__((weak)) EVP_MAC_CTX_new(EVP_MAC*); void __attribute__((weak)) EVP_MAC_CTX_free(EVP_MAC_CTX*); void __attribute__((weak)) EVP_MAC_free(EVP_MAC *mac); OSSL_PARAM __attribute__((weak)) OSSL_PARAM_construct_utf8_string(const char*, char*, size_t); OSSL_PARAM __attribute__((weak)) OSSL_PARAM_construct_end(void); int __attribute__((weak)) EVP_MAC_init(EVP_MAC_CTX*, const unsigned char*, size_t, const OSSL_PARAM[]); int __attribute__((weak)) EVP_MAC_update(EVP_MAC_CTX*, const unsigned char*, size_t); int __attribute__((weak)) EVP_MAC_final(EVP_MAC_CTX*, unsigned char*, size_t*, size_t); int __attribute__((weak)) EVP_MD_get_size(const EVP_MD*); int __attribute__((weak)) EVP_MD_size(const EVP_MD*); int __attribute__((weak)) EVP_DigestInit_ex(EVP_MD_CTX*, const EVP_MD*, ENGINE*); int __attribute__((weak)) EVP_DigestUpdate(EVP_MD_CTX*, const void*, size_t); int __attribute__((weak)) EVP_DigestFinal_ex(EVP_MD_CTX*, unsigned char*, unsigned int*); void __attribute__((weak)) EVP_MD_CTX_free(EVP_MD_CTX*); const EVP_MD* __attribute__((weak)) EVP_md5(void); EVP_MD_CTX* __attribute__((weak)) EVP_MD_CTX_new(void); const EVP_MD* __attribute__((weak)) EVP_sha256(void); __attribute__((weak)) void EVP_MD_CTX_set_flags(EVP_MD_CTX *ctx, int flags); __attribute__((weak)) int EVP_DigestFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s); __attribute__((weak)) const EVP_MD *EVP_sha1(void); __attribute__((weak)) int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in); __attribute__((weak)) int EVP_CIPHER_CTX_set_padding(EVP_CIPHER_CTX *c, int pad); __attribute__((weak)) int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr); __attribute__((weak)) EVP_CIPHER_CTX *EVP_CIPHER_CTX_new(void); __attribute__((weak)) void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *c); __attribute__((weak)) int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *c); __attribute__((weak)) int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv); __attribute__((weak)) int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, const unsigned char *in, int inl); __attribute__((weak)) int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl); __attribute__((weak)) int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv); __attribute__((weak)) int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, const unsigned char *in, int inl); __attribute__((weak)) int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *outm, int *outl); __attribute__((weak)) const EVP_CIPHER *EVP_aes_256_cbc(void); __attribute__((weak)) const EVP_CIPHER *EVP_aes_256_ctr(void); __attribute__((weak)) const EVP_CIPHER *EVP_aes_256_gcm(void); __attribute__((weak)) const EVP_CIPHER *EVP_aes_256_ecb(void); __attribute__((weak)) int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings); __attribute__((weak)) unsigned long ERR_get_error(void); __attribute__((weak)) void ERR_error_string_n(unsigned long e, char *buf, size_t len); __attribute__((weak)) int RAND_poll(void); __attribute__((weak)) int RAND_bytes(unsigned char *buf, int num); #ifdef __cplusplus } #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/trace.h000066400000000000000000000035521453617025200231270ustar00rootroot00000000000000/** * @file trace.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 University of California, Los Angeles and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Trace Macros if TILEDB_TRACE is defined */ #ifndef __TDB_TRACE_H__ #define __TDB_TRACE_H__ #include #include #include #ifdef TILEDB_TRACE # define TRACE_FN std::cerr << "Trace - Function:" << __func__ << " File:" << __FILE__ << ":" << __LINE__ << " tid=" << syscall(SYS_gettid) << std::endl << std::flush # define TRACE_FN_ARG(X) std::cerr << "Trace - Function:" << __func__ << " File:" << __FILE__ << ":" << __LINE__ << " " << X << " tid=" << syscall(SYS_gettid) << std::endl << std::flush #else # define TRACE_FN # define TRACE_FN_ARG(X) #endif #endif /*__TDB_TRACE_H__*/ genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/uri.h000066400000000000000000000046211453617025200226260ustar00rootroot00000000000000/** * @file uri.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 University of California, Los Angeles and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * URI Parsing Header File */ #ifndef URI_HH_ #define URI_HH_ #include #include struct uri { uri(const std::string& uri_s); // Accessors std::string protocol(); std::string host(); std::string port(); int16_t nport(); std::string path(); std::unordered_map query(); private: void parse(const std::string& uri_s); std::string urlDecode(const std::string& uri_s); private: std::string protocol_; std::string host_; std::string port_; int16_t nport_ = 0; std::string path_; std::unordered_map query_; }; struct azure_uri : uri { azure_uri(const std::string& uri_s); std::string account(); std::string container(); std::string endpoint(); private: std::string account_; std::string container_; std::string endpoint_; }; struct s3_uri : uri { s3_uri(const std::string& uri_s); std::string bucket(); private: std::string bucket_; }; struct gcs_uri : uri { gcs_uri(const std::string& uri_s); std::string bucket(); private: std::string bucket_; }; #endif /* URI_HH_ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/misc/utils.h000066400000000000000000000726611453617025200232000ustar00rootroot00000000000000/** * @file utils.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file contains useful (global) functions. */ #ifndef __UTILS_H__ #define __UTILS_H__ #include "storage_fs.h" #ifdef HAVE_MPI #include #endif #include #include #include #ifdef HAVE_OPENMP #include #endif #include "tiledb_constants.h" /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_UT_OK 0 #define TILEDB_UT_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_UT_ERRMSG std::string("[TileDB::utils] Error: ") /** Maximum number of bytes written in a single I/O. */ #define TILEDB_UT_MAX_WRITE_COUNT 1500000000 // ~ 1.5 GB /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_ut_errmsg; /* ********************************* */ /* FUNCTIONS */ /* ********************************* */ /** Returns true if the input is an array read mode. */ bool array_read_mode(int mode); /** Returns true if the input is an array write mode. */ bool array_write_mode(int mode); /** Returns true if the input is an array consolidate mode. */ bool array_consolidate_mode(int mode); /** * Checks if both inputs represent the '/' character. This is an auxiliary * function to adjacent_slashes_dedup(). */ bool both_slashes(char a, char b); /** * Checks if the input cell is inside the input subarray. * * @tparam T The type of the cell and subarray. * @param cell The cell to be checked. * @param subarray The subarray to be checked, expresses as [low, high] pairs * along each dimension. * @param dim_num The number of dimensions for the cell and subarray. * @return *true* if the input cell is inside the input range and * *false* otherwise. */ template bool cell_in_subarray(const T* cell, const T* subarray, int dim_num); /** * Returns the number of cells in the input subarray (considering that the * subarray is dense). * * @tparam T The type of the subarray. * @param subarray The input subarray. * @param dim_num The number of dimensions of the subarray. * @return The number of cells in the input subarray. */ template int64_t cell_num_in_subarray(const T* subarray, int dim_num); /** * Compares the precedence of two coordinates based on the column-major order. * * @tparam T The type of the input coordinates. * @param coords_a The first coordinates. * @param coords_b The second coordinates. * @param dim_num The number of dimensions of the coordinates. * @return -1 if *coords_a* precedes *coords_b*, 0 if *coords_a* and * *coords_b* are equal, and +1 if *coords_a* succeeds *coords_b*. */ template int cmp_col_order( const T* coords_a, const T* coords_b, int dim_num); /** * Compares the precedence of two coordinates associated with ids, * first on their ids (the smaller preceeds the larger) and then based * on the column-major order. * * @tparam T The type of the input coordinates. * @param id_a The id of the first coordinates. * @param coords_a The first coordinates. * @param id_b The id of the second coordinates. * @param coords_b The second coordinates. * @param dim_num The number of dimensions of the coordinates. * @return -1 if *coords_a* precedes *coords_b*, 0 if *coords_a* and * *coords_b* are equal, and +1 if *coords_a* succeeds *coords_b*. */ template int cmp_col_order( int64_t id_a, const T* coords_a, int64_t id_b, const T* coords_b, int dim_num); /** * Compares the precedence of two coordinates based on the row-major order. * * @tparam T The type of the input coordinates. * @param coords_a The first coordinates. * @param coords_b The second coordinates. * @param dim_num The number of dimensions of the coordinates. * @return -1 if *coords_a* precedes *coords_b*, 0 if *coords_a* and * *coords_b* are equal, and +1 if *coords_a* succeeds *coords_b*. */ template int cmp_row_order( const T* coords_a, const T* coords_b, int dim_num); /** * Compares the precedence of two coordinates associated with ids, * first on their ids (the smaller preceeds the larger) and then based * on the row-major order. * * @tparam T The type of the input coordinates. * @param id_a The id of the first coordinates. * @param coords_a The first coordinates. * @param id_b The id of the second coordinates. * @param coords_b The second coordinates. * @param dim_num The number of dimensions of the coordinates. * @return -1 if *coords_a* precedes *coords_b*, 0 if *coords_a* and * *coords_b* are equal, and +1 if *coords_a* succeeds *coords_b*. */ template int cmp_row_order( int64_t id_a, const T* coords_a, int64_t id_b, const T* coords_b, int dim_num); /** * Checks if a given pathURL is a supported URL * @param pathURL URL to path to be checked. * @return true if pathURL starts with a supported URL, e.g. hdfs:// or s3:// */ bool is_supported_cloud_path(const std::string& pathURL); /** * Checks if a given pathURL is azure URL supported by hdfs. * @param pathURL URL to path to be checked. * @return true if pathURL starts with wasb:// or wasbs:// */ bool is_azure_path(const std::string& pathURL); /** * Checks if a given pathURL is Azure Blob Storage. * @param pathURL URL to path to be checked. * @return true if pathURL starts with az:// */ bool is_azure_blob_storage_path(const std::string& pathURL); /** * Checks if a given pathURL is aws s3. * @param pathURL URL to path to be checked. * @return true if pathURL starts with s3:// */ bool is_s3_storage_path(const std::string& pathURL); /** * Checks if a given pathURL is GCS. * @param pathURL URL to path to be checked. * @return true if pathURL starts with gs:// */ bool is_gcs_path(const std::string& pathURL); /** * Checks if a given pathURL is HDFS. * @param pathURL URL to path to be checked. * @return true of pathURL is HDFS compliant. */ bool is_hdfs_path(const std::string& pathURL); /** * Checks if the given environment variable is set to true(case ignored) or "1" * @param name environment variable name * @return true if enviroment variable exists and is set to true or "1" */ bool is_env_set(const std::string& name); /** * Creates a new directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The name of the directory to be created. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int create_dir(StorageFS *fs, const std::string& dir); /** * Creates a new file with the given flags and mode. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file to be created. * @param flags Status and Access mode flags for the file. * @param mode Permissions for the file to be created * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int create_file(StorageFS *fs, const std::string& filename, int flags, mode_t mode); /** * Deletes a file from the filesystem * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file to be deleted. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int delete_file(StorageFS *fs, const std::string& filename); /** * Creates a special file to indicate that the input directory is a * TileDB fragment. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The name of the fragment directory where the file is created. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int create_fragment_file(StorageFS *fs, const std::string& dir); /** * Returns the directory where the program is executed. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @return The directory where the program is executed. If the program cannot * retrieve the current working directory, the empty string is returned. */ std::string current_dir(StorageFS *fs); /** * Set working directory to path * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be set as working dir * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int set_working_dir(StorageFS *fs, const std::string& dir); /** * Deletes a directory. Note that the directory must not contain other * directories, but it should only contain files. * * @param dirname The name of the directory to be deleted. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int delete_dir(StorageFS *fs, const std::string& dirname); /** * Checks if the input is a special TileDB empty value. * * @tparam T The type of the input value. * @param value The value to be checked. * @return *true* if the input value is a special TileDB empty value, and * *false* otherwise. */ template bool empty_value(T value); /** * Doubles the size of the buffer. * * @param buffer The buffer to be expanded. * @param buffer_allocated_size The original allocated size of the buffer. * After the function call, this size doubles. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int expand_buffer(void*& buffer, size_t& buffer_allocated_size); /** * Expands the input MBR so that it encompasses the input coordinates. * * @tparam T The type of the MBR and coordinates. * @param mbr The input MBR to be expanded. * @param coords The input coordinates to expand the MBR. * @param dim_num The number of dimensions of the MBR and coordinates. * @return void */ template void expand_mbr(T* mbr, const T* coords, int dim_num); /** * Returns the size of the input file. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file whose size is to be retrieved. * @return The file size on success, and TILEDB_UT_ERR for error. */ ssize_t file_size(StorageFS *fs, const std::string& filename); /** Returns the names of the directories inside the input directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The input directory. * @return The vector of directories contained in the input directory. */ std::vector get_dirs(StorageFS *fs, const std::string& dir); /** Returns the names of the files inside the input directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The input directory. * @return The vector of directories contained in the input directory. */ std::vector get_files(StorageFS *fs, const std::string& dir); /** Returns the names of the fragments inside the input directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The input directory. * @return The vector of directories that are associated with fragments contained * in the input directory. */ std::vector get_fragment_dirs(StorageFS *fs, const std::string& dir); /** * Returns the MAC address of the machine as a 12-char string, e.g., * 00332a0b8c64. Returns an empty string upon error. */ std::string get_mac_addr(); /** * GZIPs the input buffer and stores the result in the output buffer, returning * the size of compressed data. * * @param in The input buffer. * @param in_size The size of the input buffer. * @param out The output buffer. * @param out_size The available size in the output buffer. * @return The size of compressed data on success, and TILEDB_UT_ERR on error. */ ssize_t gzip( unsigned char* in, size_t in_size, unsigned char* out, size_t out_size, const int level); /** * Decompresses the GZIPed input buffer and stores the result in the output * buffer, of maximum size avail_out. * * @param in The input buffer. * @param in_size The size of the input buffer. * @param out The output buffer. * @param avail_out_size The available size in the output buffer. * @param out_size The size of the decompressed data. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int gunzip( unsigned char* in, size_t in_size, unsigned char* out, size_t avail_out_size, size_t& out_size); /** * Checks if there are duplicates in the input vector. * * @tparam T The type of the values in the input vector. * @param v The input vector. * @return *true* if the vector has duplicates, and *false* otherwise. */ template bool has_duplicates(const std::vector& v); /** * Checks if the input coordinates lie inside the input subarray. * * @tparam T The coordinates and subarray type. * @param coords The input coordinates. * @param subarray The input subarray. * @param dim_num The number of dimensions of the subarray. * @return *true* if the coordinates lie in the subarray, and *false* otherwise. */ template bool inside_subarray(const T* coords, const T* subarray, int dim_num); /** * Checks if the input vectors have common elements. * * @tparam T The type of the elements of the input vectors. * @param v1 The first input vector. * @param v2 The second input vector. * @return *true* if the input vectors have common elements, and *false* * otherwise. */ template bool intersect(const std::vector& v1, const std::vector& v2); /** * Checks if the input directory is an array. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be checked. * @return *true* if the directory is an array, and *false* otherwise. */ bool is_array(StorageFS *fs, const std::string& dir); /** * Checks if one range is fully contained in another. * * @tparam The domain type * @param range_A The first range. * @param range_B The second range. * @param dim_num The number of dimensions. * @return True if range_A is fully contained in range_B. */ template bool is_contained( const T* range_A, const T* range_B, int dim_num); /** * Checks if the input is an existing directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be checked. * @return *true* if *dir* is an existing directory, and *false* otherwise. */ bool is_dir(StorageFS *fs, const std::string& dir); /** * Checks if the input is an existing file. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param file The file to be checked. * @return tTrue* if *file* is an existing file, and *false* otherwise. */ bool is_file(StorageFS *fs, const std::string& file); /** * Checks if the input directory is a fragment. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be checked. * @return *true* if the directory is a fragment, and *false* otherwise. */ bool is_fragment(StorageFS *fs, const std::string& dir); /** * Checks if the input directory is a group. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be checked. * @return *true* if the directory is a group, and *false* otherwise. */ bool is_group(StorageFS *fs, const std::string& dir); /** * Checks if the input directory is a metadata object. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be checked. * @return *true* if the directory is a metadata object, and *false* otherwise. */ bool is_metadata(StorageFS *fs, const std::string& dir); /** Returns *true* if the input string is a positive (>0) integer number. */ bool is_positive_integer(const char* s); /** Returns *true* if the subarray contains a single element. */ template bool is_unary_subarray(const T* subarray, int dim_num); /** * Checks if the input directory is a workspace. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The directory to be checked. * @return *true* if the directory is a workspace, and *false* otherwise. */ bool is_workspace(StorageFS *fs, const std::string& dir); #ifdef HAVE_MPI /** * Reads data from a file into a buffer using MPI-IO. * * @param mpi_comm The MPI communicator. * @param filename The name of the file. * @param offset The offset in the file from which the read will start. * @param buffer The buffer into which the data will be written. * @param length The size of the data to be read from the file. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int mpi_io_read_from_file( const MPI_Comm* mpi_comm, const std::string& filaname, off_t offset, void* buffer, size_t length); /** * Syncs a file or directory using MPI-IO. If the file/directory does not exist, * the function gracefully exits (i.e., it ignores the syncing). * * @param mpi_comm The MPI communicator. * @param filename The name of the file. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int mpi_io_sync( const MPI_Comm* mpi_comm, const char* filaname); /** * Writes the input buffer to a file using MPI-IO. * * @param mpi_comm The MPI communicator. * @param filename The name of the file. * @param buffer The input buffer. * @param buffer_size The size of the input buffer. * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int mpi_io_write_to_file( const MPI_Comm* mpi_comm, const char* filename, const void* buffer, size_t buffer_size); #endif #ifdef HAVE_OPENMP /** * Destroys an OpenMP mutex. * * @param mtx The mutex to be destroyed. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_destroy(omp_lock_t* mtx); /** * Initializes an OpenMP mutex. * * @param mtx The mutex to be initialized. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_init(omp_lock_t* mtx); /** * Locks an OpenMP mutex. * * @param mtx The mutex to be locked. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_lock(omp_lock_t* mtx); /** * Unlocks an OpenMP mutex. * * @param mtx The mutex to be unlocked. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_unlock(omp_lock_t* mtx); #endif /** * Destroys a pthread mutex. * * @param mtx The mutex to be destroyed. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_destroy(pthread_mutex_t* mtx); /** * Initializes a pthread mutex. * * @param mtx The mutex to be initialized. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_init(pthread_mutex_t* mtx); /** * Locks a pthread mutex. * * @param mtx The mutex to be locked. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_lock(pthread_mutex_t* mtx); /** * Unlocks a pthread mutex. * * @param mtx The mutex to be unlocked. * @return TILEDB_UT_OK for success, and TILEDB_UT_ERR for error. */ int mutex_unlock(pthread_mutex_t* mtx); /** * Returns the parent directory of the input directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The input directory. * @return The parent directory of the input directory. */ std::string parent_dir(StorageFS *fs, const std::string& dir); /** * Reads data from a file into a buffer. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file. * @param offset The offset in the file from which the read will start. * @param buffer The buffer into which the data will be written. * @param length The size of the data to be read from the file. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int read_from_file(StorageFS *fs, const std::string& filename, off_t offset, void* buffer, size_t length); /** * Returns the absolute canonicalized directory path of the input directory. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param dir The input directory to be canonicalized. * @return The absolute canonicalized directory path of the input directory. */ std::string real_dir(StorageFS *fs, const std::string& dir); /** * Compresses with RLE. * * @param input The input buffer to be compressed. * @param input_size The size of the input buffer. * @param output The output buffer that results from compression. * @param output_allocated_size The allocated size of the output buffer. * @param value_size The size of each single value in the input buffer. * @return The size of the result ouput buffer upon success, and TILEDB_UT_ERR * on error. */ int64_t RLE_compress( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size); /** * Returns the maximum size of the output of RLE compression. * * @param input_size The input buffer size. * @param value_size The size of a sinlge value in the input buffer. * @return The maximum size of the output after RLE-compressing the input with * size input_size. */ size_t RLE_compress_bound( size_t input_size, size_t value_size); /** * Returns the maximum size of the output of RLE compression on the coordinates. * * @param input_size The input buffer size. * @param value_size The size of a sinlge value in the input buffer. * @param dim_num The number of dimensions/coordinates in a single value. * @return The maximum size of the output after RLE-compressing the input with * size input_size. */ size_t RLE_compress_bound_coords( size_t input_size, size_t value_size, int dim_num); /** * Compresses the coordinates of a buffer with RLE, assuming that the cells in * input buffer are sorted in column-major order. * * @param input The input buffer to be compressed. * @param input_size The size of the input buffer. * @param output The output buffer that results from compression. * @param output_allocated_size The allocated size of the output buffer. * @param value_size The size of each single value in the input buffer. * @param dim_num The number of dimensions/coordinates of each cell in the * input buffer. * @return The size of the result ouput buffer upon success, and TILEDB_UT_ERR * on error. */ int64_t RLE_compress_coords_col( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size, int dim_num); /** * Compresses the coordinates of a buffer with RLE, assuming that the cells in * input buffer are sorted in row-major order. * * @param input The input buffer to be compressed. * @param input_size The size of the input buffer. * @param output The output buffer that results from compression. * @param output_allocated_size The allocated size of the output buffer. * @param value_size The size of each single value in the input buffer. * @param dim_num The number of dimensions/coordinates of each cell in the * input buffer. * @return The size of the result ouput buffer upon success, and TILEDB_UT_ERR * on error. */ int64_t RLE_compress_coords_row( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size, int dim_num); /** * Decompresses with RLE. * * @param input The input buffer to be decompressed. * @param input_size The size of the input buffer. * @param output The output buffer that results from decompression. * @param output_allocated_size The allocated size of the output buffer. * @param value_size The size of each single value in the input buffer. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int RLE_decompress( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size); /** * Decompresses the coordinates of a buffer with RLE, assuming that the cells in * input buffer are sorted in column-major order. * * @param input The input buffer to be decompressed. * @param input_size The size of the input buffer. * @param output The output buffer that results from decompression. * @param output_allocated_size The allocated size of the output buffer. * @param value_size The size of each single value in the output buffer. * @param dim_num The number of dimensions/coordinates of each cell in the * output buffer. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int RLE_decompress_coords_col( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size, int dim_num); /** * Decompresses the coordinates of a buffer with RLE, assuming that the cells in * input buffer are sorted in row-major order. * * @param input The input buffer to be decompressed. * @param input_size The size of the input buffer. * @param output The output buffer that results from decompression. * @param output_allocated_size The allocated size of the output buffer. * @param value_size The size of each single value in the output buffer. * @param dim_num The number of dimensions/coordinates of each cell in the * output buffer. * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int RLE_decompress_coords_row( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size, int dim_num); /** * Checks if a string starts with a certain prefix. * * @param value The base string. * @param prefix The prefix string to be tested. * @return *true* if *value* starts with the *prefix*, and *false* otherwise. */ bool starts_with(const std::string& value, const std::string& prefix); /** * Syncs a file or directory. If the file/directory does not exist, * the function gracefully exits (i.e., it ignores the syncing). * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file. * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int sync_path(StorageFS *fs, const std::string& path); /** * Closes any open file handles associated with a file. If the file does not exist, * or if there are no open file handles it is a noop). * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file. * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int close_file(StorageFS *fs, const std::string& filename); /** * Writes the input buffer to a file. * * @param fs The storage filesystem type in use. e.g. posix, hdfs, etc. * @param filename The name of the file. * @param buffer The input buffer. * @param buffer_size The size of the input buffer. * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int write_to_file(StorageFS *fs, const std::string& filename, const void* buffer, size_t buffer_size); /** * Delete directories * * @param vector of directory paths * @return TILEDB_UT_OK on success, and TILEDB_UT_ERR on error. */ int delete_directories(StorageFS *fs, const std::vector& directories); /** * Move(Rename) Path. * * @param original path to be moved * @param name of new path * @return TILEDB_UT_OK on success and TILEDB_UT_ERR on error. */ int move_path(StorageFS *fs, const std::string& old_path, const std::string& new_path); /* * Return the TileDB empty value for the datatype */ template inline T get_tiledb_empty_value(); //Template specialization for get_tiledb_empty_value() template<> inline char get_tiledb_empty_value() { return TILEDB_EMPTY_CHAR; } template<> inline int8_t get_tiledb_empty_value() { return TILEDB_EMPTY_INT8; } template<> inline int16_t get_tiledb_empty_value() { return TILEDB_EMPTY_INT16; } template<> inline int32_t get_tiledb_empty_value() { return TILEDB_EMPTY_INT32; } template<> inline int64_t get_tiledb_empty_value() { return TILEDB_EMPTY_INT64; } template<> inline uint8_t get_tiledb_empty_value() { return TILEDB_EMPTY_UINT8; } template<> inline uint16_t get_tiledb_empty_value() { return TILEDB_EMPTY_UINT16; } template<> inline uint32_t get_tiledb_empty_value() { return TILEDB_EMPTY_UINT32; } template<> inline uint64_t get_tiledb_empty_value() { return TILEDB_EMPTY_UINT64; } template<> inline float get_tiledb_empty_value() { return TILEDB_EMPTY_FLOAT32; } template<> inline double get_tiledb_empty_value() { return TILEDB_EMPTY_FLOAT64; } #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/000077500000000000000000000000001453617025200223645ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_azure_blob.h000066400000000000000000000167631453617025200264220ustar00rootroot00000000000000/** * @file storage_azure_blob.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2019-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Azure Blob Storage Interface * */ #ifndef __STORAGE_AZURE_BLOB_H__ #define __STORAGE_AZURE_BLOB_H__ #include "storage_fs.h" #include "adls_client.h" #include "base64.h" #include "blob/blob_client.h" #include "storage_account.h" #include "storage_credential.h" #include "tiledb_constants.h" #include #include #include using namespace azure::storage_lite; class AzureBlob : public StorageCloudFS { public: AzureBlob(const std::string& home); std::string current_dir(); int set_working_dir(const std::string& dir); std::string real_dir(const std::string& dir); int create_dir(const std::string& dir); int delete_dir(const std::string& dir); std::vector get_dirs(const std::string& dir); std::vector get_files(const std::string& dir); int create_file(const std::string& filename, int flags, mode_t mode); int delete_file(const std::string& filename); ssize_t file_size(const std::string& filename); int read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length); int write_to_file(const std::string& filename, const void *buffer, size_t buffer_size); protected: std::shared_ptr blob_client_ = nullptr; std::shared_ptr bc_wrapper_ = nullptr; blob_client_wrapper *blob_client_wrapper_ = nullptr; std::string account_name_; std::string container_name_; std::string working_dir_; std::shared_ptr adls_client_ = nullptr; std::mutex write_map_mtx_; std::unordered_map> write_map_; std::unordered_map filesizes_map_; std::vector> empty_metadata; private: struct membuf: std::streambuf { membuf(const void *buffer, size_t buffer_size, std::ios_base::openmode mode = std::ios_base::in) { char *p_buffer(reinterpret_cast(const_cast(buffer))); if (mode == std::ios_base::in) { this->setg(p_buffer, p_buffer, p_buffer+buffer_size); } else if (mode == std::ios_base::out) { this->setp(p_buffer, p_buffer+buffer_size); } else { throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "No membuf support for openmodes other than in and out"); } } pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override { if (mode == std::ios_base::in) { if (dir == std::ios_base::cur) { this->gbump(off); } else if (dir == std::ios_base::end) { this->setg(this->eback(), this->egptr() + off, this->egptr()); } else if (dir == std::ios_base::beg) { this->setg(this->eback(), this->eback() + off, this->egptr()); } return this->gptr() - this->eback(); } // Support only input modes for now return std::streampos(std::streamoff(-1)); } pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override { return seekoff(pos - pos_type(off_type(0)), std::ios_base::beg, mode); } }; struct imemstream: virtual membuf, std::istream { imemstream(const void *buffer, size_t buffer_size) : membuf(buffer, buffer_size, std::ios_base::in), std::istream(static_cast(this)) { } }; struct omemstream: virtual membuf, std::ostream { omemstream(const void *buffer, size_t buffer_size) : membuf(buffer, buffer_size, std::ios_base::out), std::ostream(static_cast(this)) { } }; size_t max_stream_size_ = 1024; int num_threads_ = 1; // use TILEDB_NUM_THREADS if needed std::string get_path(const std::string& path); std::vector generate_block_ids(const std::string& path, int num_blocks) { std::vector block_ids; block_ids.reserve(num_blocks); int existing_num_blocks = 0; const std::lock_guard lock(write_map_mtx_); auto search = write_map_.find(path); if (search == write_map_.end()) { write_map_.insert({path, std::vector{}}); search = write_map_.find(path); assert (search != write_map_.end()); } else { existing_num_blocks = search->second.size(); } for (int i = existing_num_blocks; i < existing_num_blocks+num_blocks; i++) { std::string block_id = std::to_string(i); block_id = std::string(12 - block_id.length(), '-') + block_id; block_id = to_base64(reinterpret_cast(block_id.data()), block_id.length()); auto block = put_block_list_request_base::block_item{block_id, put_block_list_request_base::block_type::uncommitted}; block_ids.emplace_back(block_id); search->second.push_back(std::move(block)); } if (existing_num_blocks+num_blocks > 50000) { // https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs - A block blob can include up to 50,000 blocks std::cerr << "Block Blobs cannot be comprised of more than 50000 blocks. " << "Current number of blocks=" << std::to_string(existing_num_blocks+num_blocks) << std::endl; return {}; } return block_ids; } std::future> upload_block_blob(const std::string &blob, uint64_t block_size, int num_blocks, std::vector block_list, const char* buffer, uint64_t bufferlen, int parallelism=1); void update_expected_filesizes_map(const std::string& path, size_t size) { auto search = filesizes_map_.find(path); if (search == filesizes_map_.end()) { filesizes_map_.insert({path, size}); } else { filesizes_map_[path] += size; } } ssize_t expected_filesize_from_map(const std::string& path) { auto search = filesizes_map_.find(path); if (search == filesizes_map_.end()) { return -1; } else { return search->second; } } bool path_exists(const std::string& path); int create_path(const std::string& path); int commit_file(const std::string& filename); }; #endif /* __STORAGE_AZURE_BLOB_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_buffer.h000066400000000000000000000140131453617025200255310ustar00rootroot00000000000000/** * @file storage_buffer.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020-2022 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the StorageBuffer class that buffers writes/reads to/from files */ #pragma once #include "storage_fs.h" #include "storage_posixfs.h" #include "tiledb_constants.h" #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_BF_OK 0 #define TILEDB_BF_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_BF_ERRMSG std::string("[TileDB::StorageBuffer] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_bf_errmsg; class StorageBuffer { public: /** * Constructor that accepts StorageFS and the filename minimally. StorageBuffer is a no-op * if the upload/download limits are not set in StorageFS. */ StorageBuffer(StorageFS *fs, const std::string& filename, size_t chunk_size, const bool is_read=false); virtual ~StorageBuffer() { free_buffer(); } /** * Reads data from the cached buffer with file offsets maintained implicitly. * If the cached buffer does not contain the segment, its read from the file into * the cache. The size of the cache is guarded by StorageFS download size limit. * @param bytes The buffer into which the data will be written. * @param size The size of the data to be read from the cached buffer. */ virtual int read_buffer(void *bytes, size_t size); /** * Reads the data from the cached buffer from the offset into bytes. * If the cached buffer does not contain the segment for the offset and size, * its read from the file into the cache. The size of the cache is guarded by * StorageFS download size limit. * @param offset The offset from which the data in the buffer will be read from. * @param bytes The buffer into which the data will be written. * @param size The size of the data to be read from the cached buffer. */ int read_buffer(off_t offset, void *bytes, size_t size); /** * Appends given data from the buffer backed by bytes into the file. The buffer will * actually be written out only after reaching the StorageFS upload size limit. Ownership * of the buffer is relinquished to the caller on return. * @param bytes The buffer from which data will be read to append to file */ int append_buffer(const void *bytes, size_t size); /** * Persist bytes in the buffer to the FileSystem. It is the responsibilty of the client to * check upload file size thresholds for Cloud Storage as only the last block to be uploaded * can be lower than the threshold. See * https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-uploads * https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html * Currently used internally only with CompressedStorageBuffer. */ inline int flush() { return read_only_?TILEDB_FS_OK:write_buffer(); } /** * Finalize flushes existing buffers and releases any allocated memory. */ virtual int finalize(); protected: void *buffer_ = NULL; size_t buffer_size_ = 0; off_t buffer_offset_ = 0; size_t allocated_buffer_size_ = 0; StorageFS *fs_ = NULL; const std::string filename_; size_t filesize_ = -1; // Relevant for only read_only size_t file_offset_ = 0; const bool read_only_; size_t chunk_size_ = 0; bool is_error_ = false; /** * Frees the allocated cached buffers and reinitializes all associated variables. */ virtual void free_buffer() { if (buffer_) free(buffer_); buffer_ = NULL; buffer_offset_ = 0; buffer_size_ = 0; } int read_buffer(); virtual int write_buffer(); }; class CompressedStorageBuffer : public StorageBuffer { public: CompressedStorageBuffer(StorageFS *fs, const std::string& filename, size_t chunk_size, const bool is_read=false, const int compression_type=TILEDB_NO_COMPRESSION, const int compression_level=0) : StorageBuffer(fs, filename, chunk_size, is_read), compression_type_(compression_type), compression_level_(compression_level) { } ~CompressedStorageBuffer() { free_buffer(); } int read_buffer(void *bytes, size_t size); int write_buffer(); int finalize(); void free_buffer() { if (compress_buffer_) free(compress_buffer_); compress_buffer_ = NULL; compress_buffer_size_ = 0; StorageBuffer::free_buffer(); } private: const int compression_type_; const int compression_level_; void *compress_buffer_ = NULL; size_t compress_buffer_size_ = 0; std::shared_ptr compressed_write_buffer_ = 0; size_t compressed_write_buffer_size_ = 0; int initialize_gzip_stream(z_stream *strm); int gzip_read_buffer(); int gzip_write_buffer(); }; genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_fs.h000066400000000000000000000150701453617025200246740ustar00rootroot00000000000000/** * @file storage_fs.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2019 Omics Data Automation Inc. and Intel Corporation * @copyright Copyright (c) 2020-2021 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Storage API exposes some filesystem specific functionality implemented in TileDB. * */ #ifndef __STORAGE_FS_H__ #define __STORAGE_FS_H__ #include "uri.h" #include #include #include #include #include #include /**@{*/ /** Return code. */ #define TILEDB_FS_OK 0 #define TILEDB_FS_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_FS_ERRMSG std::string("[TileDB::FileSystem] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_fs_errmsg; /** Base Class for Filesystems */ class StorageFS { public: virtual ~StorageFS(); virtual std::string current_dir() = 0; virtual int set_working_dir(const std::string& dir) = 0; virtual bool is_dir(const std::string& dir) = 0; virtual bool is_file(const std::string& file) = 0; virtual std::string real_dir(const std::string& dir) = 0; virtual int create_dir(const std::string& dir) = 0; virtual int delete_dir(const std::string& dir) = 0; virtual std::vector get_dirs(const std::string& dir) = 0; virtual std::vector get_files(const std::string& dir) = 0; virtual int create_file(const std::string& filename, int flags, mode_t mode) = 0; virtual int delete_file(const std::string& filename) = 0; virtual ssize_t file_size(const std::string& filename) = 0; virtual int read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length) = 0; virtual int write_to_file(const std::string& filename, const void *buffer, size_t buffer_size) = 0; virtual int move_path(const std::string& old_path, const std::string& new_path) = 0; virtual int sync_path(const std::string& path) = 0; virtual int close_file(const std::string& filename); virtual bool locking_support(); size_t get_download_buffer_size() { auto env_var = getenv("TILEDB_DOWNLOAD_BUFFER_SIZE"); if (env_var) { return std::stoull(env_var); } else { return download_buffer_size_; } } size_t get_upload_buffer_size() { auto env_var = getenv("TILEDB_UPLOAD_BUFFER_SIZE"); if (env_var) { return std::stoull(env_var); } else { return upload_buffer_size_; } } void set_download_buffer_size(size_t buffer_size) { download_buffer_size_ = buffer_size; } void set_upload_buffer_size(size_t buffer_size) { upload_buffer_size_ = buffer_size; } static inline std::string slashify(const std::string& path) { if (path.empty()) { return "/"; } else if (path.back() != '/') { return path + '/'; } else { return path; } } static inline std::string unslashify(const std::string& path) { if (!path.empty() && path.back() == '/') { return path.substr(0, path.size()-1); } else { return path; } } static inline std::string append_paths(const std::string& path1, const std::string& path2) { return slashify(path1) + path2; } size_t download_buffer_size_ = 0; size_t upload_buffer_size_ = 0; }; class StorageCloudFS : public virtual StorageFS { #define DELIMITER "/" public: int create_dir(const std::string& dir) { // no-op return TILEDB_FS_OK; } bool is_dir(const std::string& dir) { if (get_path(dir).length() == 0) { // This must be the container - OK return true; } return path_exists(slashify(dir)); } bool is_file(const std::string& file) { return path_exists(unslashify(file)); } int move_path(const std::string& old_path, const std::string& new_path) { throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "TBD: No support for moving path"); } int sync_path(const std::string& path); int close_file(const std::string& filename); protected: std::string get_path(const std::string& path); virtual bool path_exists(const std::string& path) = 0; virtual int create_path(const std::string& path) = 0; virtual int commit_file(const std::string& filename) = 0; #ifdef __linux__ // Possible certificate files; stop after finding one. // https://github.com/golang/go/blob/f0e940ebc985661f54d31c8d9ba31a553b87041b/src/crypto/x509/root_linux.go#L7 const std::vector possible_ca_certs_locations = { "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 "/etc/ssl/ca-bundle.pem", // OpenSUSE "/etc/pki/tls/cacert.pem", // OpenELEC "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 "/etc/ssl/cert.pem", // Alpine Linux }; std::string locate_ca_certs() { for (const std::string& location : possible_ca_certs_locations) { struct stat st; memset(&st, 0, sizeof(struct stat)); if (!stat(location.c_str(), &st) && S_ISREG(st.st_mode)) { return location; } } #ifdef DEBUG std::cerr << "CA Certs path not located. Using system defaults" << std::endl; #endif return ""; } #else std::string locate_ca_certs() { return ""; } #endif protected: std::string working_dir_; }; #endif /* __STORAGE_FS_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_gcs.h000066400000000000000000000065131453617025200250420ustar00rootroot00000000000000/** * @ storage_gcs.h * * @section LICENSE * * The MIT License * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * GCS derived from StorageFS for Google Cloud Services * */ #ifndef __STORAGE_GCS_H__ #define __STORAGE_GCS_H__ #include "tiledb_constants.h" #include #include #include #include "storage_fs.h" /* avoid warnings of the type "include/absl/meta/type_traits.h:293:36: warning: builtin __has_trivial_destructor is deprecated; use __is_trivially_destructible instead [-Wdeprecated-builtins] : std::integral_constant namespace gcs = google::cloud::storage; using ::google::cloud::StatusOr; class GCS : public StorageCloudFS { public: GCS(const std::string& home); ~GCS(); std::string get_path(const std::string& path) { return StorageCloudFS::get_path(path); } std::string current_dir(); int set_working_dir(const std::string& dir); std::string real_dir(const std::string& dir); int create_dir(const std::string& dir); int delete_dir(const std::string& dir); std::vector get_dirs(const std::string& dir); std::vector get_files(const std::string& dir); int create_file(const std::string& filename, int flags, mode_t mode); int delete_file(const std::string& filename); ssize_t file_size(const std::string& filename); int read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length); int write_to_file(const std::string& filename, const void *buffer, size_t buffer_size); protected: std::string bucket_name_; StatusOr client_; std::mutex write_map_mtx_; typedef struct multipart_upload_info_t { public: size_t part_number_ = 0; size_t last_uploaded_size_ = 0; } multipart_upload_info_t; std::unordered_map write_map_; bool path_exists(const std::string& path); int create_path(const std::string& path); int delete_path(const std::string& path); int commit_file(const std::string& filename); }; #endif /* __STORAGE_GCS_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_hdfs.h000066400000000000000000000055431453617025200252140ustar00rootroot00000000000000/** * @file storage_hdfs.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 University of California, Los Angeles and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * HDFS derived from StorageFS for HDFS * */ #ifndef __STORAGE_HDFS_H__ #define __STORAGE_HDFS_H__ #ifdef USE_HDFS #include "storage_fs.h" #include "hdfs.h" #include "tiledb_constants.h" #include #include class HDFS : public StorageFS { public: HDFS(const std::string& home); ~HDFS(); std::string current_dir(); int set_working_dir(const std::string& dir); bool is_dir(const std::string& dir); bool is_file(const std::string& file); std::string real_dir(const std::string& dir); int create_dir(const std::string& dir); int delete_dir(const std::string& dir); std::vector get_dirs(const std::string& dir); std::vector get_files(const std::string& dir); int create_file(const std::string& filename, int flags, mode_t mode); int delete_file(const std::string& filename); ssize_t file_size(const std::string& filename); int read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length); int write_to_file(const std::string& filename, const void *buffer, size_t buffer_size); int move_path(const std::string& old_path, const std::string& new_path); int sync_path(const std::string& path); int close_file(const std::string& filename); bool locking_support(); protected: hdfsFS hdfs_handle_ = NULL; std::mutex read_map_mtx_, write_map_mtx_; std::unordered_map read_map_, write_map_; std::unordered_map read_count_; }; #endif /* USE_HDFS */ #endif /* __STORAGE_HDFS_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_manager.h000066400000000000000000001011021453617025200256660ustar00rootroot00000000000000/** * @file storage_manager.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class StorageManager. */ #ifndef __STORAGE_MANAGER_H__ #define __STORAGE_MANAGER_H__ #include "array.h" #include "array_iterator.h" #include "array_schema.h" #include "array_schema_c.h" #include "metadata.h" #include "metadata_iterator.h" #include "metadata_schema_c.h" #include "storage_manager_config.h" #include #ifdef HAVE_OPENMP #include #endif #include #include /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_SM_OK 0 #define TILEDB_SM_ERR -1 /**@}*/ /**@{*/ /** Lock types. */ #define TILEDB_SM_SHARED_LOCK 0 #define TILEDB_SM_EXCLUSIVE_LOCK 1 /**@}*/ /** Name of the consolidation file lock. */ #define TILEDB_SM_CONSOLIDATION_FILELOCK_NAME ".__consolidation_lock" /** Default error message. */ #define TILEDB_SM_ERRMSG std::string("[TileDB::StorageManager] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_sm_errmsg; /** * The storage manager, which is repsonsible for creating, deleting, etc. of * TileDB objects (i.e., workspaces, groups, arrays and metadata). */ class StorageManager { public: /* ********************************* */ /* TYPE DEFINITIONS */ /* ********************************* */ /** Implements an open array entry. */ class OpenArray; /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ StorageManager(); /** Destructor. */ ~StorageManager(); /* ********************************* */ /* MUTATORS */ /* ********************************* */ /** * Finalizes the storage manager, properly freeing memory. * * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int finalize(); /** * Initializes the storage manager. This function creates the TileDB home * directory, which by default is "~/.tiledb/". If the user home directory * cannot be retrieved, then the TileDB home directory is set to the current * working directory. * * @param config The configuration parameters. If it is NULL, * then the default TileDB parameters are used. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int init(StorageManagerConfig* config); /** * Retrieve the configuration associated with the storage manager. * @return pointer to StorageManagerConfig. */ StorageManagerConfig* get_config(); /* ********************************* */ /* WORKSPACE */ /* ********************************* */ /** * Creates a TileDB workspace. * * @param workspace The directory of the workspace to be created. This * directory should not be inside another TileDB workspace, group, array * or metadata directory. * @return TILEDB_SM_OK for succes, and TILEDB_SM_ERR for error. */ int workspace_create(const std::string& workspace); /** * Lists all TileDB workspaces, copying their directory names in the input * string buffers. * * @param workspaces An array of strings that will store the listed * workspaces. Note that this should be pre-allocated by the user. If the * size of each string is smaller than the corresponding workspace path * name, the function will probably segfault. It is a good idea to * allocate to each workspace string TILEDB_NAME_MAX_LEN characters. * @param workspace_num The number of allocated elements of the *workspaces* * string array. After the function execution, this will hold the actual * number of workspaces written in the *workspaces* string array. If the * number of allocated elements is smaller than the number of existing * TileDB workspaces, the function will return an error. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int ls_workspaces( const char *parent_dir, char** workspaces, int& workspace_num); /** * Counts the number of TileDB workspaces. * * @param workspace_num The number of TileDB workspace to be returned. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int ls_workspaces_c(const char *parent_dir, int& workspace_num); /* ********************************* */ /* GROUP */ /* ********************************* */ /** * Creates a new TileDB group. * * @param group The directory of the group to be created in the file system. * This should be a directory whose parent is a TileDB workspace or * another TileDB group. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int group_create(const std::string& group) const; /* ********************************* */ /* ARRAY */ /* ********************************* */ /** * Consolidates the fragments of an array into a single fragment. * * @param array_dir The name of the array to be consolidated. * @param buffer_size The size of buffers for reading/writing attributes during consolidation. * @param batch size Consolidation will occur batch-wise with a smaller batch_size set of * fragments getting consolidating together. If <=0, all fragments will be consolidated together. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_consolidate(const char* array_dir, size_t buffer_size, int batch_size); /** * Creates a new TileDB array. * * @param array_schema_c The array schema. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_create(const ArraySchemaC* array_schema_c) const; /** * Creates a new TileDB array. * * @param array_schema The array schema. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_create(const ArraySchema* array_schema) const; /** * Loads the schema of an array from the disk, allocating appropriate memory * space for it. * * @param array_dir The directory of the array. * @param array_schema The schema to be loaded. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int array_load_schema( const char* array_dir, ArraySchema*& array_schema) const; /** * Initializes a TileDB array. * * @param array The array object to be initialized. The function * will allocate memory space for it. * @param array_dir The directory of the array to be initialized. * @param mode The mode of the array. It must be one of the following: * - TILEDB_ARRAY_WRITE * - TILEDB_ARRAY_WRITE_UNSORTED * - TILEDB_ARRAY_READ * - TILEDB_ARRAY_READ_SORTED_COL * - TILEDB_ARRAY_READ_SORTED_ROW * @param subarray The subarray in which the array read/write will be * constrained on. If it is NULL, then the subarray is set to the entire * array domain. For the case of writes, this is meaningful only for * dense arrays, and specifically dense writes. * @param attributes A subset of the array attributes the read/write will be * constrained on. A NULL value indicates **all** attributes (including * the coordinates in the case of sparse arrays). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int array_init( Array*& array, const char* array_dir, int mode, const void* subarray, const char** attributes, int attribute_num); /** * Finalizes an array, properly freeing the memory space. * * @param array The array to be finalized. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int array_finalize(Array* array); /** * Syncs all currently written files in the input array. * * @param array The array to be synced. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int array_sync(Array* array); /** * Syncs the currently written files associated with the input attribute * in the input array. * * @param array The array to be synced. * @param attribute The name of the attribute to be synced. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int array_sync_attribute( Array* array, const std::string& attribute); /** * Initializes an array iterator for reading cells, potentially constraining * it on a subset of attributes, as well as a subarray. The cells will be read * in the order they are stored on the disk, maximizing performance. * * @param array_it The TileDB array iterator to be created. The * function will allocate the appropriate memory space for the iterator. * @param array The directory of the array the iterator is initialized for. * @param mode The read mode, which can be one of the following: * - TILEDB_ARRAY_READ\n * Reads the cells in the native order they are stored on the disk. * - TILEDB_ARRAY_READ_SORTED_COL\n * Reads the cells in column-major order within the specified subarray. * - TILEDB_ARRAY_READ_SORTED_ROW\n * Reads the cells in column-major order within the specified subarray. * @param subarray The subarray in which the array iterator will be * constrained on. If it is NULL, then the subarray is set to the entire * array domain. * @param attributes A subset of the array attributes the iterator will be * constrained on. A NULL value indicates **all** attributes (including * the coordinates in the case of sparse arrays). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param buffers This is an array of buffers similar to tiledb_array_read. * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read cells. The iterator will * read from the disk the relevant cells in batches, by fitting as many * cell values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding sizes (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many cells as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @param filter_expression An expression string that evaluates to a boolean * to allow for cells to be filtered out from the buffers while reading. * If NULL or empty, no filter is applied. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int array_iterator_init( ArrayIterator*& array_it, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes, const char* filter_expression); /** * Finalizes an array iterator, properly freeing the allocating memory space. * * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int array_iterator_finalize(ArrayIterator* array_it); /* ********************************* */ /* METADATA */ /* ********************************* */ /** * Consolidates the fragments of a metadata object into a single fragment. * * @param metadata_dir The name of the metadata to be consolidated. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int metadata_consolidate(const char* metadata_dir); /** * Creates a new TileDB metadata object. * * @param metadata_schema_c The metadata schema. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int metadata_create(const MetadataSchemaC* metadata_schema_c) const; /** * Creates a new TileDB metadata object. * * @param array_schema The metadata (array) schema. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int metadata_create(const ArraySchema* array_schema) const; /** * Loads the schema of a metadata object from the disk, allocating appropriate * memory space for it. * * @param metadata_dir The directory of the metadata. * @param array_schema The schema to be loaded. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int metadata_load_schema( const char* metadata_dir, ArraySchema*& array_schema) const; /** * Initializes a TileDB metadata object. * * @param metadata The metadata object to be initialized. The function * will allocate memory space for it. * @param metadata_dir The directory of the metadata. * @param mode The mode of the metadata. It must be one of the following: * - TILEDB_METADATA_WRITE * - TILEDB_METADATA_READ * @param attributes A subset of the metadata attributes the read/write will * be constrained on. A NULL value indicates **all** attributes (including * the key as an extra attribute in the end). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int metadata_init( Metadata*& metadata, const char* metadata_dir, int mode, const char** attributes, int attribute_num); /** * Finalizes a TileDB metadata object, properly freeing the memory space. * * @param metadata The metadata to be finalized. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int metadata_finalize(Metadata* metadata); /** * Initializes a metadata iterator, potentially constraining it * on a subset of attributes. The values will be read in the order they are * stored on the disk, maximizing performance. * * @param metadata_it The TileDB metadata iterator to be created. The * function will allocate the appropriate memory space for the iterator. * @param metadata_dir The directory of the metadata the iterator is * initialized for. * @param attributes A subset of the metadata attributes the iterator will be * constrained on. A NULL value indicates **all** attributes (including * the key as an extra attribute in the end). * @param attribute_num The number of the input attributes. If *attributes* is * NULL, then this should be set to 0. * @param buffers This is an array of buffers similar to tiledb_metadata_read. * It is the user that allocates and provides buffers that the iterator * will use for internal buffering of the read values. The iterator will * read from the disk the values in batches, by fitting as many * values as possible in the user buffers. This gives the user the * flexibility to control the prefetching for optimizing performance * depending on the application. * @param buffer_sizes The corresponding sizes (in bytes) of the allocated * memory space for *buffers*. The function will prefetch from the * disk as many cells as can fit in the buffers, whenever it finishes * iterating over the previously prefetched data. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int metadata_iterator_init( MetadataIterator*& metadata_it, const char* metadata_dir, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes); /** * Finalizes the iterator, properly freeing the allocating memory space. * * @param metadata_it The TileDB metadata iterator. * @return TILEDB_SM_OK on success, and TILEDB_SM_ERR on error. */ int metadata_iterator_finalize(MetadataIterator* metadata_it); /* ********************************* */ /* MISC */ /* ********************************* */ /** * Lists all the TileDB objects in a directory, copying them into the input * buffers. * * @param parent_dir The parent directory of the TileDB objects to be listed. * @param dirs An array of strings that will store the listed TileDB objects. * Note that the user is responsible for allocating the appropriate memory * space for this array of strings. A good idea for each string length is * to set is to TILEDB_NAME_MAX_LEN. * @param dir_types The types of the corresponding TileDB objects, which can * be the following (they are self-explanatory): * - TILEDB_WORKSPACE * - TILEDB_GROUP * - TILEDB_ARRAY * - TILEDB_METADATA * @param dir_num The number of elements allocated by the user for *dirs*. * After the function terminates, this will hold the actual number of * TileDB objects that were stored in *dirs*. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int ls( const char* parent_dir, char** dirs, int* dir_types, int& dir_num) const; /** * Counts the TileDB objects in a directory. * * @param parent_dir The parent directory of the TileDB objects to be listed. * @param dir_num The number of TileDB objects to be returned. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int ls_c(const char* parent_dir, int& dir_num) const; /** * Clears all TileDB objects contained in the directory. * * @param parent_dir The parent directory of the TileDB objects to be deleted. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int clear_contained_artifacts(const std::string& parent_dir) const; /** * Clears a TileDB directory. The corresponding TileDB object (workspace, * group, array, or metadata) will still exist after the execution of the * function, but it will be empty (i.e., as if it was just created). * * @param dir The directory to be cleared. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int clear(const std::string& dir) const; /** * Deletes a TileDB directory (workspace, group, array, or metadata) entirely. * * @param dir The directory to be deleted. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int delete_entire(const std::string& dir); /** * Moves a TileDB directory (workspace, group, array or metadata). * * @param old_dir The old directory. * @param new_dir The new directory. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int move(const std::string& old_dir, const std::string& new_dir); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** The TileDB configuration parameters. */ StorageManagerConfig* config_; /** The Filesystem associated with this configuration */ StorageFS* fs_; /** OpenMP mutex for creating/deleting an OpenArray object. */ #ifdef HAVE_OPENMP omp_lock_t open_array_omp_mtx_; #endif /** Pthread mutex for creating/deleting an OpenArray object. */ pthread_mutex_t open_array_pthread_mtx_; /** Stores the currently open arrays. */ std::map open_arrays_; /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ /** * Clears a TileDB array. The array will still exist after the execution of * the function, but it will be empty (i.e., as if it was just created). * * @param array The array to be cleared. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_clear(const std::string& array) const; /** * Decrements the number of times the input array is initialized. If this * number reaches 0, the it deletes the open array entry (and hence clears * the schema and fragment book-keeping of the array). * * @param array The array name. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_close(const std::string& array); /** * Deletes a TileDB array entirely. * * @param array The array to be deleted. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_delete(const std::string& array) const; /** * Gets the names of the existing fragments of an array. * * @param array The input array. * @param fragment_names The fragment names to be returned. * @return void */ void array_get_fragment_names( const std::string& array, std::vector& fragment_names); /** * Gets an open array entry for the array being initialized. If this * is the first time the array is initialized, then the function creates * a new open array entry for this array. * * @param array The array name. * @param open_array The open array entry to be returned. * @param flag that's set to true if this array is opened for the first time in this object * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_get_open_array_entry( const std::string& array, OpenArray*& open_array, bool& opened_first_time); /** * Loads the book-keeping structures of all the fragments of an array from the * disk, allocating appropriate memory space for them. * * @param array_schema The array schema. * @param fragment_names The names of the fragments of the array. * @param book_keeping The book-keeping structures to be returned. * @param mode The array mode * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int array_load_book_keeping( const ArraySchema* array_schema, const std::vector& fragment_names, std::vector& book_keeping, int mode); /** * Moves a TileDB array. * * @param old_array The old array directory. * @param new_array The new array directory. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_move( const std::string& old_array, const std::string& new_array) const; /** * Opens an array. This creates or updates an OpenArray entry for this array, * and loads the array schema and book-keeping if it is the first time this * array is being initialized. The book-keeping structures are loaded only * if the input mode is a read mode. * * @param array_name The array name (must be absolute path). * @param mode The mode in which the array is being initialized. * @param open_array The open array entry that is retrieved. * @param mode The array mode. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_open( const std::string& array_name, OpenArray*& open_array, int mode); /** * Stores the input array schema into the input array directory (serializing * it into a sequence of bytes and storing it in a binary file). * * @param dir The array directory where the array schema is stored. * @param array_schema The array schema to be stored. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int array_store_schema( const std::string& dir, const ArraySchema* array_schema) const; /** * It sets the TileDB configuration parameters. * * @param config The configuration parameters. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int config_set(StorageManagerConfig* config); /** * Creates a special file that serves as lock needed for implementing * thread- and process-safety during consolidation. The file is * created inside an array or metadata directory. * * @param dir The array or metadata directory the filelock is created for. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int consolidation_filelock_create(const std::string& dir) const; /** * Locks the consolidation file lock. * * @param array_name The name of the array the lock is applied on. * @param fd The file descriptor of the filelock. * @param lock_type The lock type, which can be either TILEDB_SM_SHARED_LOCK * or TILEDB_SM_EXCLUSIVE_LOCK. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int consolidation_filelock_lock( const std::string& array_name, int& fd, int lock_type) const; /** * Unlocks the consolidation file lock. * * @param fd The file descriptor of the filelock. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int consolidation_filelock_unlock(int fd) const; /** * Finalizes the consolidation process, applying carefully the locks so that * this can be done concurrently with other reads. The function finalizes the * new consolidated fragment, and deletes the old fragments that created it. * * @param new_fragment The new consolidated fragment that the function will * finalize. * @param old_fragment_names The names of the old fragments that need to be * deleted. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int consolidation_finalize( Fragment* new_fragment, const std::vector& old_fragment_names); /** * Creates a special group file inside the group directory. * * @param dir The group directory. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int create_group_file(const std::string& dir) const; /** * Creates a special workspace file inside the workpace directory. * * @param workspace The workspace directory. * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int create_workspace_file(const std::string& workspace) const; /** * Clears a TileDB group. The group will still exist after the execution of * the function, but it will be empty (i.e., as if it was just created). * * @param group The group to be cleared. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int group_clear(const std::string& group) const; /** * Deletes a TileDB group entirely. * * @param group The group to be deleted. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int group_delete(const std::string& group) const; /** * Moves a TileDB group. * * @param old_group The old group directory. * @param new_group The new group directory. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int group_move( const std::string& old_group, const std::string& new_group) const; /** * Clears a TileDB metadata object. The metadata will still exist after the * execution of the function, but it will be empty (i.e., as if it was just * created). * * @param metadata The metadata to be cleared. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int metadata_clear(const std::string& metadata) const; /** * Deletes a TileDB metadata object entirely. * * @param metadata The metadata to be deleted. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int metadata_delete(const std::string& metadata) const; /** * Moves a TileDB metadata object. * * @param old_metadata The old metadata directory. * @param new_metadata The new metadata directory. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int metadata_move( const std::string& old_metadata, const std::string& new_metadata) const; /** * Destroys open array the mutexes. * * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int open_array_mtx_destroy(); /** * Initializes open array mutexes. * * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int open_array_mtx_init(); /** * Locks open array mutexes. * * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int open_array_mtx_lock(); /** * Unlocks open array mutexes. * * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int open_array_mtx_unlock(); /** * Appropriately sorts the fragment names based on their name timestamps. * The result is stored in the input vector. * * @param fragment_names The fragment names to be sorted. This will also hold * the result of the function after termination. * @return void */ void sort_fragment_names(std::vector& fragment_names) const; /** * Clears a TileDB workspace. The workspace will still exist after the * execution of the function, but it will be empty (i.e., as if it was just * created). * * @param workspace The workspace to be cleared. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int workspace_clear(const std::string& workspace) const; /** * Deletes a TileDB workspace entirely. * * @param workspace The workspace to be deleted. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int workspace_delete(const std::string& workspace); /** * Moves a TileDB workspace. * * @param old_workspace The old workspace directory. * @param new_workspace The new workspace directory. * @return TILEDB_SM_OK for success and TILEDB_SM_ERR for error. */ int workspace_move( const std::string& old_workspace, const std::string& new_workspace); }; /** * Stores information about an open array. An array is open if it has been * initialized once (withour being finalized). The difference with array * initialization is that an array can be initialized multiple times, * but opened only once. This structure maintains the information that * can be used by multiple array objects that initialize the same array, * in order to avoid replication and speed-up performance (e.g., array * schema and book-keeping). */ class StorageManager::OpenArray { public: // ATTRIBUTES /** The array schema. */ ArraySchema* array_schema_; /** The book-keeping structures for all the fragments of the array. */ std::vector book_keeping_; /** * A counter for the number of times the array has been initialized after * it was opened. */ int cnt_; /** Descriptor for the consolidation filelock. */ int consolidation_filelock_; /** The names of the fragments of the open array. */ std::vector fragment_names_; /** * An OpenMP mutex used to lock the array when loading the array schema and * the book-keeping structures from the disk. */ #ifdef HAVE_OPENMP omp_lock_t omp_mtx_; #endif /** * A pthread mutex used to lock the array when loading the array schema and * the book-keeping structures from the disk. */ pthread_mutex_t pthread_mtx_; // FUNCTIONS /** * Destroys the mutexes. * * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int mutex_destroy(); /** * Initializes the mutexes. * * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int mutex_init(); /** * Locks the mutexes. * * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int mutex_lock(); /** * Unlocks the mutexes. * * @return TILEDB_SM_OK for success, and TILEDB_SM_ERR for error. */ int mutex_unlock(); }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_manager_config.h000066400000000000000000000137701453617025200272300ustar00rootroot00000000000000/** * @file storage_manager_config.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines class StorageManagerConfig. */ #ifndef __CONFIG_H__ #define __CONFIG_H__ #ifdef HAVE_MPI #include #endif #include #include "storage_fs.h" #include "storage_posixfs.h" #include "storage_hdfs.h" #include "storage_gcs.h" #include "tiledb_constants.h" /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /**@{*/ /** Return code. */ #define TILEDB_SMC_OK 0 #define TILEDB_SMC_ERR -1 /**@}*/ /** Default error message. */ #define TILEDB_SMC_ERRMSG std::string("[TileDB::StorageManagerConfig] Error: ") /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ /** Stores potential error messages. */ extern std::string tiledb_smc_errmsg; /** * This class is responsible for the TileDB storage manager configuration * parameters. */ class StorageManagerConfig { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ /** Constructor. */ StorageManagerConfig(); /** Destructor. */ ~StorageManagerConfig(); /* ********************************* */ /* MUTATORS */ /* ********************************* */ #ifdef HAVE_MPI /** * Initializes the configuration parameters. * * @param home The TileDB home directory. * @param mpi_comm The MPI communicator. * @param read_method The method for reading data from a file. * It can be one of the following: * - TILEDB_IO_READ * TileDB will use POSIX read. * - TILEDB_IO_MMAP * TileDB will use mmap. * - TILEDB_IO_MPI * TileDB will use MPI-IO read. * @param write_method The method for writing data to a file. * It can be one of the following: * - TILEDB_IO_WRITE * TileDB will use POSIX write. * - TILEDB_IO_MPI * TileDB will use MPI-IO write. * @param enable_shared_posixfs_optimizations in POSIX fs if set * @return void. */ int init( const char* home, MPI_Comm* mpi_comm = NULL, int read_method = TILEDB_IO_READ, int write_methods = TILEDB_IO_WRITE, const bool enable_shared_posixfs_optimizations = false); #else /** * Initializes the configuration parameters. * * @param home The TileDB home directory. * @param read_method The method for reading data from a file. * It can be one of the following: * - TILEDB_IO_READ * TileDB will use POSIX read. * - TILEDB_IO_MMAP * TileDB will use mmap. * - TILEDB_IO_MPI * TileDB will use MPI-IO read. * @param write_method The method for writing data to a file. * It can be one of the following: * - TILEDB_IO_WRITE * TileDB will use POSIX write. * - TILEDB_IO_MPI * TileDB will use MPI-IO write. * @param enable_shared_posixfs_optimizations if set * @return void. */ int init( const char* home, int read_method = TILEDB_IO_READ, int write_method = TILEDB_IO_WRITE, const bool enable_shared_posixfs_optimizations = false); #endif /* ********************************* */ /* ACCESSORS */ /* ********************************* */ /** Returns the TileDB home directory. */ const std::string& home() const; #ifdef HAVE_MPI /** Returns the MPI communicator. */ MPI_Comm* mpi_comm() const; #endif /** Returns the read method. */ int read_method() const; /** Returns the write method. */ int write_method() const; /** Returns the supporting filesystem */ StorageFS* get_filesystem() const; private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ /** TileDB home directory. */ std::string home_; #ifdef HAVE_MPI /** The MPI communicator. */ MPI_Comm* mpi_comm_; #endif /** * The method for reading data from a file. * It can be one of the following: * - TILEDB_IO_READ * TileDB will use POSIX read. * - TILEDB_IO_MMAP * TileDB will use mmap. * - TILEDB_IO_MPI * TileDB will use MPI-IO read. */ int read_method_; /** * The method for writing data to a file. * It can be one of the following: * - TILEDB_IO_WRITE * TileDB will use POSIX write. * - TILEDB_IO_MPI * TileDB will use MPI-IO write. */ int write_method_; /** The Filesystem type associated with this configuration */ StorageFS *fs_ = NULL; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_posixfs.h000066400000000000000000000060201453617025200257520ustar00rootroot00000000000000/** * @file local_fs.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * LocalFS derived from StorageFS for Posix Filesystem * */ #ifndef __STORAGE_POSIXFS_H__ #define __STORAGE_POSIXFS_H__ #include "storage_fs.h" #include #include class PosixFS : public StorageFS { public: ~PosixFS(); std::string current_dir(); int set_working_dir(const std::string& dir); bool is_dir(const std::string& dir); bool is_file(const std::string& file); std::string real_dir(const std::string& dir); int create_dir(const std::string& dir); int delete_dir(const std::string& dir); std::vector get_dirs(const std::string& dir); std::vector get_files(const std::string& dir); int create_file(const std::string& filename, int flags, mode_t mode); int delete_file(const std::string& filename); ssize_t file_size(const std::string& filename); int read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length); int write_to_file(const std::string& filename, const void *buffer, size_t buffer_size); int move_path(const std::string& old_path, const std::string& new_path); int sync_path(const std::string& path); bool locking_support(); int close_file(const std::string& filename); void set_keep_write_file_handles_open(const bool val); bool keep_write_file_handles_open(); void set_disable_file_locking(const bool val); bool disable_file_locking(); private: std::mutex write_map_mtx_; std::unordered_map write_map_; bool keep_write_file_handles_open_set_ = false; bool keep_write_file_handles_open_ = false; bool is_disable_file_locking_set = false; bool disable_file_locking_ = false; int write_to_file_keep_file_handles_open(const std::string& filename, const void *buffer, size_t buffer_size); }; #endif /* __STORAGE_POSIXFS_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/include/storage/storage_s3.h000066400000000000000000000067571453617025200246250ustar00rootroot00000000000000/** * @file storage_s3.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * AWS S3 Support for StorageFS */ #ifndef __STORAGE_S3_H__ #define __STORAGE_S3_H__ #include "storage_fs.h" #include #include #include #include #include #include #include class S3 : public StorageCloudFS { public: S3(const std::string& home); ~S3(); std::string get_path(const std::string& path) { return StorageCloudFS::get_path(path); } std::string current_dir(); int set_working_dir(const std::string& dir); std::string real_dir(const std::string& dir); int create_dir(const std::string& dir); int delete_dir(const std::string& dir); std::vector get_dirs(const std::string& dir); std::vector get_files(const std::string& dir); int create_file(const std::string& filename, int flags, mode_t mode); int delete_file(const std::string& filename); ssize_t file_size(const std::string& filename); int read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length); int write_to_file(const std::string& filename, const void *buffer, size_t buffer_size); // Helper functions inline Aws::String to_aws_string(const std::string& s) const { return Aws::String(s.begin(), s.end()); } protected: std::string bucket_name_; std::shared_ptr client_; std::mutex write_map_mtx_; typedef struct multipart_upload_info_t { public: multipart_upload_info_t(const std::string& upload_id) : upload_id_(upload_id) { completed_parts_ = std::make_shared(); } std::string upload_id_; size_t part_number_ = 0; size_t last_uploaded_size_ = 0; std::shared_ptr completed_parts_; bool abort_upload_ = false; } multipart_upload_info_t; std::unordered_map write_map_; bool path_exists(const std::string& path); int create_path(const std::string& path); int delete_path(const std::string& path); int commit_file(const std::string& filename); }; #endif /* __STORAGE_S3_H__ */ genomicsdb-0.0~git20231212.9d7ddd0/core/src/000077500000000000000000000000001453617025200200645ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/000077500000000000000000000000001453617025200212025ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/array.cc000066400000000000000000001231021453617025200226260ustar00rootroot00000000000000/** * @file array.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2019, 2022-2023 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the Array class. */ #include "array.h" #include "mem_utils.h" #include "utils.h" #include #include #include #include #include #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_AR_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_ar_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ Array::Array() { array_read_state_ = NULL; array_sorted_read_state_ = NULL; array_sorted_write_state_ = NULL; array_schema_ = NULL; subarray_ = NULL; expression_ = NULL; aio_thread_created_ = false; array_clone_ = NULL; } Array::~Array() { // Applicable to both arrays and array clones std::vector::iterator it = fragments_.begin(); for(; it != fragments_.end(); ++it) if(*it != NULL) delete *it; if(expression_ != NULL) delete expression_; if(array_read_state_ != NULL) delete array_read_state_; if(array_sorted_read_state_ != NULL) delete array_sorted_read_state_; if(array_sorted_write_state_ != NULL) delete array_sorted_write_state_; // Applicable only to non-clones if(array_clone_ != NULL) { delete array_clone_; if(array_schema_ != NULL) delete array_schema_; } if(subarray_ != NULL) free(subarray_); subarray_ = NULL; } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ void Array::aio_handle_requests() { // Holds the next AIO request AIO_Request* aio_next_request; // Initiate infinite loop for(;;) { // Lock AIO mutext if(pthread_mutex_lock(&aio_mtx_)) { std::string errmsg = "Cannot lock AIO mutex"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return; } // If the thread is canceled, unblock and exit if(aio_thread_canceled_) { if(pthread_mutex_unlock(&aio_mtx_)) PRINT_ERROR("Cannot unlock AIO mutex while canceling AIO thread"); else aio_thread_created_ = false; return; } // Wait for AIO requests while(aio_queue_.size() == 0) { // Wait to be signaled if(pthread_cond_wait(&aio_cond_, &aio_mtx_)) { PRINT_ERROR("Cannot wait on AIO mutex condition"); return; } // If the thread is canceled, unblock and exit if(aio_thread_canceled_) { if(pthread_mutex_unlock(&aio_mtx_)) { std::string errmsg = "Cannot unlock AIO mutex while canceling AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; } else { aio_thread_created_ = false; } return; } } // Pop the next AIO request aio_next_request = aio_queue_.front(); aio_queue_.pop(); // Unlock AIO mutext if(pthread_mutex_unlock(&aio_mtx_)) { std::string errmsg = "Cannot unlock AIO mutex"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return; } // Handle the next AIO request aio_handle_next_request(aio_next_request); // Set last handled AIO request aio_last_handled_request_ = aio_next_request->id_; } } int Array::aio_read(AIO_Request* aio_request) { // Sanity checks if(!read_mode()) { std::string errmsg = "Cannot (async) read from array; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Create the AIO thread if not already done if(!aio_thread_created_) if(aio_thread_create() != TILEDB_AR_OK) return TILEDB_ERR; // Push the AIO request in the queue if(aio_push_request(aio_request) != TILEDB_AR_OK) return TILEDB_AR_ERR; // Success return TILEDB_AR_OK; } int Array::aio_write(AIO_Request* aio_request) { // Sanity checks if(!write_mode()) { std::string errmsg = "Cannot (async) write to array; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Create the AIO thread if not already done if(!aio_thread_created_) if(aio_thread_create() != TILEDB_AR_OK) return TILEDB_ERR; // Push the AIO request in the queue if(aio_push_request(aio_request) != TILEDB_AR_OK) return TILEDB_AR_ERR; // Success return TILEDB_AR_OK; } Array* Array::array_clone() const { return array_clone_; } const ArraySchema* Array::array_schema() const { return array_schema_; } const std::vector& Array::attribute_ids() const { return attribute_ids_; } const StorageManagerConfig* Array::config() const { return config_; } int Array::fragment_num() const { return fragments_.size(); } std::vector Array::fragments() const { return fragments_; } int Array::mode() const { return mode_; } bool Array::overflow() const { // Not applicable to writes if(!read_mode()) return false; // Check overflow if(array_sorted_read_state_ != NULL) return array_sorted_read_state_->overflow(); else return array_read_state_->overflow(); } bool Array::overflow(int attribute_id) const { assert(read_mode() || consolidate_mode()); // Trivial case if(fragments_.size() == 0) return false; // Check overflow if(array_sorted_read_state_ != NULL) return array_sorted_read_state_->overflow(attribute_id); else return array_read_state_->overflow(attribute_id); } int Array::read(void** buffers, size_t* buffer_sizes, size_t* skip_counts) { // Sanity checks if(!read_mode() && !consolidate_mode()) { std::string errmsg = "Cannot read from array; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Check if there are no fragments int buffer_i = 0; int attribute_id_num = attribute_ids_.size(); if(fragments_.size() == 0) { for(int i=0; ivar_size(attribute_ids_[i])) ++buffer_i; else buffer_i += 2; } return TILEDB_AR_OK; } // Handle sorted modes if(mode_ == TILEDB_ARRAY_READ_SORTED_COL || mode_ == TILEDB_ARRAY_READ_SORTED_ROW) { if(skip_counts) { tiledb_ar_errmsg = "skip counts only handled for TILDB_ARRAY_READ mode, unsupported for TILEDB_ARRAY_READ_SORTED* modes"; return TILEDB_AR_ERR; } if(array_sorted_read_state_->read(buffers, buffer_sizes) == TILEDB_ASRS_OK) { return TILEDB_AR_OK; } else { tiledb_ar_errmsg = tiledb_asrs_errmsg; return TILEDB_AR_ERR; } } else { // mode_ == TILDB_ARRAY_READ return read_default(buffers, buffer_sizes, skip_counts); } } int Array::evaluate_cell(void** buffer, size_t* buffer_sizes, int64_t* positions) { if (expression_) { int rc = expression_->evaluate_cell(buffer, buffer_sizes, positions); if (rc == TILEDB_EXPR_ERR) { tiledb_ar_errmsg = tiledb_expr_errmsg; return TILEDB_AR_ERR; } return rc; } return true; } int Array::read_default(void** buffers, size_t* buffer_sizes, size_t* skip_counts) { if(array_read_state_->read(buffers, buffer_sizes, skip_counts) != TILEDB_ARS_OK) { tiledb_ar_errmsg = tiledb_ars_errmsg; return TILEDB_AR_ERR; } // Success return TILEDB_AR_OK; } bool Array::read_mode() const { return array_read_mode(mode_); } const void* Array::subarray() const { return subarray_; } bool Array::write_mode() const { return array_write_mode(mode_); } bool Array::consolidate_mode() const { return array_consolidate_mode(mode_); } /* ****************************** */ /* MUTATORS */ /* ****************************** */ Fragment* get_fragment_for_consolidation(StorageFS *fs, std::string fragment_name, const Array *array) { Fragment* fragment = new Fragment(array); bool dense = !fs->is_file(fs->append_paths(fragment_name, std::string(TILEDB_COORDS) + TILEDB_FILE_SUFFIX)); BookKeeping *book_keeping = new BookKeeping(array->array_schema(), dense, fragment_name, TILEDB_ARRAY_READ); if (book_keeping->load(fs) != TILEDB_BK_OK) { tiledb_ar_errmsg = tiledb_bk_errmsg; return NULL; } if(fragment->init(fragment_name, book_keeping, TILEDB_ARRAY_READ) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return NULL; } return fragment; } int Array::consolidate( Fragment*& new_fragment, std::vector& old_fragment_names, size_t buffer_size, int batch_size) { // Trivial case if(fragment_names_.size() <= 1) return TILEDB_AR_OK; #ifdef DO_MEMORY_PROFILING std::cerr << "Using buffer_size=" << buffer_size << " for consolidation" << std::endl; std::cerr << "Number of fragments to consolidate=" << fragment_names_.size() << std::endl; print_memory_stats("beginning consolidation"); #endif // Consolidate on a per-batch and per-attribute basis if (batch_size <= 0 || (size_t)batch_size > fragment_names_.size()) { batch_size = fragment_names_.size(); } auto remaining = (fragment_names_.size()%batch_size); int num_batches = (fragment_names_.size()/batch_size)+(remaining?1:0); // Create the buffers int attribute_num = array_schema_->attribute_num(); int var_attribute_num = array_schema_->var_attribute_num(); int buffer_num = attribute_num + 1 + var_attribute_num; void **buffers = (void**) malloc(buffer_num * sizeof(void*)); size_t *buffer_sizes = (size_t*) malloc(buffer_num * sizeof(size_t)); void *buffer = malloc(buffer_size); void *buffer_var = malloc(buffer_size); StorageFS* fs = config_->get_filesystem(); // Consolidating per batch std::string last_batch_fragment_name; for (auto batch=0; batchnew_fragment_name(); if(new_fragment_name == "") { std::string errmsg = "Cannot produce new fragment name"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } new_fragment = new Fragment(this); if(new_fragment->init(new_fragment_name, TILEDB_ARRAY_WRITE, subarray_) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } int actual_batch_size; if (batch == num_batches-1) { actual_batch_size = remaining?remaining:batch_size; } else { actual_batch_size = batch_size; } auto start = batch*actual_batch_size; for (auto fragment_i = start; fragment_iattribute_num()+1; ++i,++j) { buffers[j] = buffer; buffer_sizes[j] = buffer_size; if (array_schema_->var_size(i)) { buffers[j+1] = buffer_var; buffer_sizes[j+1] = buffer_size; j++; } if(consolidate(new_fragment, i, buffers, buffer_sizes, buffer_size) != TILEDB_AR_OK) { delete_dir(fs, new_fragment->fragment_name()); delete new_fragment; return TILEDB_AR_ERR; } #ifdef DO_MEMORY_PROFILING print_memory_stats("End: consolidating attribute " + array_schema_->attribute(i)); #endif trim_memory(); } // Cleanup after batch consolidation delete array_read_state_; array_read_state_ = NULL; for (auto fragment_i = 0u; fragment_ifinalize(); delete fragments_[fragment_i]->book_keeping(); delete fragments_[fragment_i]; } fragments_.clear(); if (batch < (num_batches-1)) { new_fragment->finalize(); last_batch_fragment_name = new_fragment->fragment_name(); old_fragment_names.push_back(last_batch_fragment_name); new_fragment = NULL; } #ifdef DO_MEMORY_PROFILING print_memory_stats("End: batch " + std::to_string(batch+1) + "/" + std::to_string(num_batches)); #endif trim_memory(); } // Clean up free(buffer_var); free(buffer); free(buffer_sizes); free(buffers); #ifdef DO_MEMORY_PROFILING print_memory_stats("after final consolidation"); #endif old_fragment_names.insert(std::end(old_fragment_names), std::begin(fragment_names_), std::end(fragment_names_)); // Success return TILEDB_AR_OK; } int Array::consolidate( Fragment* new_fragment, int attribute_id, void **buffers, size_t *buffer_sizes, size_t buffer_size) { // For easy reference int attribute_num = array_schema_->attribute_num(); // Do nothing if the array is dense for the coordinates attribute if(array_schema_->dense() && attribute_id == attribute_num) return TILEDB_AR_OK; // Cache the buffer indices associated with the attribute int buffer_index = -1; int buffer_var_index = -1; int buffer_i = 0; for(int i=0; ivar_size(i)) { assert(buffers[buffer_i]); buffer_var_index = buffer_i; ++buffer_i; } } else { buffers[buffer_i] = NULL; buffer_sizes[buffer_i] = 0; ++buffer_i; if(array_schema_->var_size(i)) { buffers[buffer_i] = NULL; buffer_sizes[buffer_i] = 0; ++buffer_i; } } } #ifdef DO_MEMORY_PROFILING print_memory_stats("after alloc for attribute="+array_schema_->attribute(attribute_id)); #endif // Read and write attribute until there is no overflow int rc_write = TILEDB_FG_OK; int rc_read = TILEDB_FG_OK; do { // Set or reset buffer sizes as they are modified by the reads buffer_sizes[buffer_index] = buffer_size; if (buffer_var_index != -1) { buffer_sizes[buffer_var_index] = buffer_size; } // Read rc_read = read(buffers, buffer_sizes); if(rc_read != TILEDB_FG_OK) break; // Write rc_write = new_fragment->write( (const void**) buffers, (const size_t*) buffer_sizes); if(rc_write != TILEDB_FG_OK) break; } while(overflow(attribute_id)); // Error if(rc_write != TILEDB_FG_OK || rc_read != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } // Success return TILEDB_AR_OK; } int Array::finalize() { // Initializations int rc = TILEDB_FG_OK; int fragment_num = fragments_.size(); bool fg_error = false; for(int i=0; ifinalize(); if(rc != TILEDB_FG_OK) fg_error = true; delete fragments_[i]; } fragments_.clear(); // Clean the array read state if(array_read_state_ != NULL) { delete array_read_state_; array_read_state_ = NULL; } // Clean the array sorted read state if(array_sorted_read_state_ != NULL) { delete array_sorted_read_state_; array_sorted_read_state_ = NULL; } // Clean the array sorted write state if(array_sorted_write_state_ != NULL) { delete array_sorted_write_state_; array_sorted_write_state_ = NULL; } if (consolidate_mode()) { return fg_error?TILEDB_AR_ERR:TILEDB_AR_OK; } // Clean the AIO-related members int rc_aio_thread = aio_thread_destroy(); int rc_aio_cond = TILEDB_AR_OK, rc_aio_mtx = TILEDB_AR_OK; if(pthread_cond_destroy(&aio_cond_)) rc_aio_cond = TILEDB_AR_ERR; if(pthread_mutex_destroy(&aio_mtx_)) rc_aio_mtx = TILEDB_AR_ERR; while(aio_queue_.size() != 0) { free(aio_queue_.front()); aio_queue_.pop(); } // Finalize the clone int rc_clone = TILEDB_AR_OK; if(array_clone_ != NULL) rc_clone = array_clone_->finalize(); // Errors if(rc != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } if(rc_aio_thread != TILEDB_AR_OK) return TILEDB_AR_ERR; if(rc_aio_cond != TILEDB_AR_OK) { std::string errmsg = "Cannot destroy AIO mutex condition"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } if(rc_aio_mtx != TILEDB_AR_OK) { std::string errmsg = "Cannot destroy AIO mutex"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } if(rc_clone != TILEDB_AR_OK) return TILEDB_AR_ERR; if(fg_error) return TILEDB_AR_ERR; // Success return TILEDB_AR_OK; } int Array::init( const ArraySchema* array_schema, const std::string array_path_used, const std::vector& fragment_names, const std::vector& book_keeping, int mode, const char** attributes, int attribute_num, const void* subarray, const StorageManagerConfig* config, Array* array_clone) { // Set mode mode_ = mode; //Set path used to access array - might be different from the one in the schema array_path_used_ = array_path_used; // Set array clone array_clone_ = array_clone; // Sanity check on mode if(!read_mode() && !write_mode() && !consolidate_mode()) { std::string errmsg = "Cannot initialize array; Invalid array mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Set config config_ = config; // Set array schema array_schema_ = array_schema; if (consolidate_mode()) { fragment_names_ = fragment_names; } // Set subarray size_t subarray_size = 2*array_schema_->coords_size(); subarray_ = malloc(subarray_size); if(subarray == NULL) memcpy(subarray_, array_schema->domain(), subarray_size); else memcpy(subarray_, subarray, subarray_size); // Get attributes std::vector attributes_vec; if(attributes == NULL) { // Default: all attributes attributes_vec = array_schema_->attributes(); if(array_schema_->dense() && mode != TILEDB_ARRAY_WRITE_UNSORTED) // Remove coordinates attribute for dense arrays, // unless in TILEDB_WRITE_UNSORTED mode attributes_vec.pop_back(); } else { // Custom attributes // Get attributes bool coords_found = false; bool sparse = !array_schema_->dense(); for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } attributes_vec.push_back(attributes[i]); if(!strcmp(attributes[i], TILEDB_COORDS)) coords_found = true; } // Sanity check on duplicates if(has_duplicates(attributes_vec)) { std::string errmsg = "Cannot initialize array; Duplicate attributes"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // For the case of the clone sparse array, append coordinates if they do // not exist already if(sparse && array_clone == NULL && !coords_found && !is_metadata(config_->get_filesystem(), get_array_path_used())) attributes_vec.push_back(TILEDB_COORDS); } // Set attribute ids if(array_schema_->get_attribute_ids(attributes_vec, attribute_ids_) != TILEDB_AS_OK) { tiledb_ar_errmsg = tiledb_as_errmsg; return TILEDB_AR_ERR; } try { // Initialize new fragment if needed if(write_mode()) { // WRITE MODE // Get new fragment name std::string new_fragment_name = this->new_fragment_name(); if(new_fragment_name == "") { std::string errmsg = "Cannot produce new fragment name"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Create new fragment Fragment* fragment = new Fragment(this); fragments_.push_back(fragment); if(fragment->init(new_fragment_name, mode_, subarray) != TILEDB_FG_OK) { array_schema_ = NULL; tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } // Create ArraySortedWriteState if(mode_ == TILEDB_ARRAY_WRITE_SORTED_COL || mode_ == TILEDB_ARRAY_WRITE_SORTED_ROW) { array_sorted_write_state_ = new ArraySortedWriteState(this); if(array_sorted_write_state_->init() != TILEDB_ASWS_OK) { tiledb_ar_errmsg = tiledb_asws_errmsg; delete array_sorted_write_state_; array_sorted_write_state_ = NULL; return TILEDB_AR_ERR; } } else { array_sorted_write_state_ = NULL; } } else if (consolidate_mode()) { array_read_state_ = NULL; array_sorted_read_state_ = NULL; return TILEDB_OK; } else { // READ MODE // Open fragments if(open_fragments(fragment_names, book_keeping) != TILEDB_AR_OK) { array_schema_ = NULL; return TILEDB_AR_ERR; } // Create ArrayReadState array_read_state_ = new ArrayReadState(this); // Create ArraySortedReadState if(mode_ != TILEDB_ARRAY_READ) { array_sorted_read_state_ = new ArraySortedReadState(this); if(array_sorted_read_state_->init() != TILEDB_ASRS_OK) { tiledb_ar_errmsg = tiledb_asrs_errmsg; delete array_sorted_read_state_; array_sorted_read_state_ = NULL; return TILEDB_AR_ERR; } } else { array_sorted_read_state_ = NULL; } } // Initialize the AIO-related members aio_cond_ = PTHREAD_COND_INITIALIZER; if(pthread_mutex_init(&aio_mtx_, NULL)) { std::string errmsg = "Cannot initialize AIO mutex"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } if(pthread_cond_init(&aio_cond_, NULL)) { std::string errmsg = "Cannot initialize AIO mutex condition"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } aio_thread_canceled_ = false; aio_thread_created_ = false; aio_last_handled_request_ = -1; } catch(std::system_error& ex) { PRINT_ERROR(ex.what()); tiledb_ar_errmsg = "Array initialization failed"; PRINT_ERROR(tiledb_ar_errmsg); return TILEDB_AR_ERR; } // Return return TILEDB_AR_OK; } int Array::apply_filter(const char* filter_expression) { // Set up filter expression if (filter_expression != NULL && strlen(filter_expression) > 0) { std::vector attributes_vec; for (std::vector::iterator it = attribute_ids_.begin(); it != attribute_ids_.end(); it++) { attributes_vec.push_back(array_schema_->attribute(*it)); } expression_ = new Expression(filter_expression); if (expression_->init(attribute_ids_, array_schema_)) { tiledb_ar_errmsg = tiledb_expr_errmsg; return TILEDB_AR_ERR; } } return TILEDB_AR_OK; } int Array::reset_attributes( const char** attributes, int attribute_num) { // Get attributes std::vector attributes_vec; if(attributes == NULL) { // Default: all attributes attributes_vec = array_schema_->attributes(); if(array_schema_->dense()) // Remove coordinates attribute for dense attributes_vec.pop_back(); } else { // Custom attributes // Copy attribute names for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } attributes_vec.push_back(attributes[i]); } // Sanity check on duplicates if(has_duplicates(attributes_vec)) { std::string errmsg = "Cannot reset attributes; Duplicate attributes"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } } // Set attribute ids if(array_schema_->get_attribute_ids(attributes_vec, attribute_ids_) != TILEDB_AS_OK) { tiledb_ar_errmsg = tiledb_as_errmsg; return TILEDB_AR_ERR; } // Reset subarray so that the read/write states are flushed if(reset_subarray(subarray_) != TILEDB_AR_OK) return TILEDB_AR_ERR; // Success return TILEDB_AR_OK; } int Array::reset_subarray(const void* subarray) { // Sanity check assert(read_mode() || write_mode()); // For easy referencd int fragment_num = fragments_.size(); // Finalize fragments if in write mode if(write_mode()) { // Finalize and delete fragments for(int i=0; ifinalize(); delete fragments_[i]; } fragments_.clear(); } // Set subarray size_t subarray_size = 2*array_schema_->coords_size(); if(subarray_ == NULL) subarray_ = malloc(subarray_size); if(subarray == NULL) memcpy(subarray_, array_schema_->domain(), subarray_size); else memmove(subarray_, subarray, subarray_size); //subarray_ and subarray might be the same memory area - see line 780 // Re-set or re-initialize fragments if(write_mode()) { // WRITE MODE // Finalize last fragment if(fragments_.size() != 0) { assert(fragments_.size() == 1); if(fragments_[0]->finalize() != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } delete fragments_[0]; fragments_.clear(); } // Re-initialize ArraySortedWriteState if(array_sorted_write_state_ != NULL) delete array_sorted_write_state_; if(mode_ == TILEDB_ARRAY_WRITE_SORTED_COL || mode_ == TILEDB_ARRAY_WRITE_SORTED_ROW) { array_sorted_write_state_ = new ArraySortedWriteState(this); if(array_sorted_write_state_->init() != TILEDB_ASWS_OK) { tiledb_ar_errmsg = tiledb_asws_errmsg; delete array_sorted_write_state_; array_sorted_write_state_ = NULL; return TILEDB_AR_ERR; } } else { array_sorted_write_state_ = NULL; } // Get new fragment name std::string new_fragment_name = this->new_fragment_name(); if(new_fragment_name == "") { std::string errmsg = "Cannot generate new fragment name"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Create new fragment Fragment* fragment = new Fragment(this); fragments_.push_back(fragment); if(fragment->init(new_fragment_name, mode_, subarray) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } } else { // READ MODE // Re-initialize the read state of the fragments for(int i=0; ireset_read_state(); // Re-initialize array read state if(array_read_state_ != NULL) { delete array_read_state_; array_read_state_ = NULL; } array_read_state_ = new ArrayReadState(this); // Re-initialize ArraySortedReadState if(array_sorted_read_state_ != NULL) delete array_sorted_read_state_; if(mode_ != TILEDB_ARRAY_READ) { array_sorted_read_state_ = new ArraySortedReadState(this); if(array_sorted_read_state_->init() != TILEDB_ASRS_OK) { tiledb_ar_errmsg = tiledb_asrs_errmsg; delete array_sorted_read_state_; array_sorted_read_state_ = NULL; return TILEDB_AR_ERR; } } else { array_sorted_read_state_ = NULL; } } // Success return TILEDB_AR_OK; } int Array::reset_subarray_soft(const void* subarray) { // Sanity check assert(read_mode() || write_mode()); // For easy referencd int fragment_num = fragments_.size(); // Finalize fragments if in write mode if(write_mode()) { // Finalize and delete fragments for(int i=0; ifinalize(); delete fragments_[i]; } fragments_.clear(); } // Set subarray size_t subarray_size = 2*array_schema_->coords_size(); if(subarray_ == NULL) subarray_ = malloc(subarray_size); if(subarray == NULL) memcpy(subarray_, array_schema_->domain(), subarray_size); else memcpy(subarray_, subarray, subarray_size); // Re-set or re-initialize fragments if(write_mode()) { // WRITE MODE // Do nothing } else { // READ MODE // Re-initialize the read state of the fragments for(int i=0; ireset_read_state(); // Re-initialize array read state if(array_read_state_ != NULL) { delete array_read_state_; array_read_state_ = NULL; } array_read_state_ = new ArrayReadState(this); } // Success return TILEDB_AR_OK; } int Array::sync() { // Sanity check if(!write_mode()) { std::string errmsg = "Cannot sync array; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Sanity check assert(fragments_.size() == 1); // Sync fragment if(fragments_[0]->sync() != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } else { return TILEDB_AR_OK; } } int Array::sync_attribute(const std::string& attribute) { // Sanity checks if(!write_mode()) { std::string errmsg = "Cannot sync attribute; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Sanity check assert(fragments_.size() == 1); // Sync fragment if(fragments_[0]->sync_attribute(attribute) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } else { return TILEDB_AR_OK; } } int Array::write(const void** buffers, const size_t* buffer_sizes) { // Sanity checks if(!write_mode()) { std::string errmsg = "Cannot write to array; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Write based on mode if(mode_ == TILEDB_ARRAY_WRITE_SORTED_COL || mode_ == TILEDB_ARRAY_WRITE_SORTED_ROW) { if (array_sorted_write_state_->write(buffers, buffer_sizes) != TILEDB_ASWS_OK) { tiledb_ar_errmsg = tiledb_asws_errmsg; return TILEDB_AR_ERR; } } else if(mode_ == TILEDB_ARRAY_WRITE || mode_ == TILEDB_ARRAY_WRITE_UNSORTED) { if (write_default(buffers, buffer_sizes) != TILEDB_AR_OK) { return TILEDB_AR_ERR; } } else { assert(0); } // In all modes except TILEDB_ARRAY_WRITE, the fragment must be finalized if(mode_ != TILEDB_ARRAY_WRITE) { if(fragments_[0]->finalize() != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } delete fragments_[0]; fragments_.clear(); } // Success return TILEDB_AR_OK; } int Array::write_default(const void** buffers, const size_t* buffer_sizes) { // Sanity checks if(!write_mode()) { std::string errmsg = "Cannot write to array; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Create and initialize a new fragment if(fragments_.size() == 0) { // Get new fragment name std::string new_fragment_name = this->new_fragment_name(); if(new_fragment_name == "") { std::string errmsg = "Cannot produce new fragment name"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Create new fragment Fragment* fragment = new Fragment(this); fragments_.push_back(fragment); if(fragment->init(new_fragment_name, mode_, subarray_) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } } // Dispatch the write command to the new fragment if(fragments_[0]->write(buffers, buffer_sizes) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } // Success return TILEDB_AR_OK; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ void Array::aio_handle_next_request(AIO_Request* aio_request) { int rc = TILEDB_AR_OK; if(read_mode()) { // READ MODE // Invoke the read if(aio_request->mode_ == TILEDB_ARRAY_READ) { // Reset the subarray only if this request does not continue from the last if(aio_last_handled_request_ != aio_request->id_) reset_subarray_soft(aio_request->subarray_); // Read rc = read_default(aio_request->buffers_, aio_request->buffer_sizes_); } else { // This may initiate a series of new AIO requests // Reset the subarray hard this time (updating also the subarray // of the ArraySortedReadState object. if(aio_last_handled_request_ != aio_request->id_) reset_subarray(aio_request->subarray_); // Read rc = read(aio_request->buffers_, aio_request->buffer_sizes_); } } else { // WRITE MODE // Invoke the write if(aio_request->mode_ == TILEDB_ARRAY_WRITE || aio_request->mode_ == TILEDB_ARRAY_WRITE_UNSORTED) { // Reset the subarray only if this request does not continue from the last if(aio_last_handled_request_ != aio_request->id_) reset_subarray_soft(aio_request->subarray_); // Write rc = write_default( (const void**) aio_request->buffers_, (const size_t*) aio_request->buffer_sizes_); } else { // This may initiate a series of new AIO requests // Reset the subarray hard this time (updating also the subarray // of the ArraySortedWriteState object. if(aio_last_handled_request_ != aio_request->id_) reset_subarray(aio_request->subarray_); // Write rc = write( (const void**) aio_request->buffers_, (const size_t*) aio_request->buffer_sizes_); } } if(rc == TILEDB_AR_OK) { // Success // Check for overflow (applicable only to reads) if(aio_request->mode_ == TILEDB_ARRAY_READ && array_read_state_->overflow()) { *aio_request->status_= TILEDB_AIO_OVERFLOW; if(aio_request->overflow_ != NULL) { for(int i=0; ioverflow_[i] = array_read_state_->overflow(attribute_ids_[i]); } } else if((aio_request->mode_ == TILEDB_ARRAY_READ_SORTED_COL || aio_request->mode_ == TILEDB_ARRAY_READ_SORTED_ROW ) && array_sorted_read_state_->overflow()) { *aio_request->status_= TILEDB_AIO_OVERFLOW; if(aio_request->overflow_ != NULL) { for(int i=0; ioverflow_[i] = array_sorted_read_state_->overflow(attribute_ids_[i]); } } else { // Completion *aio_request->status_= TILEDB_AIO_COMPLETED; } // Invoke the callback if(aio_request->completion_handle_ != NULL) (*(aio_request->completion_handle_))(aio_request->completion_data_); } else { // Error *aio_request->status_= TILEDB_AIO_ERR; } } void *Array::aio_handler(void* context) { // This will enter an indefinite loop that will handle all incoming AIO // requests ((Array*) context)->aio_handle_requests(); // Return return NULL; } int Array::aio_push_request(AIO_Request* aio_request) { // Set the request status *aio_request->status_ = TILEDB_AIO_INPROGRESS; // Lock AIO mutex if(pthread_mutex_lock(&aio_mtx_)) { std::string errmsg = "Cannot lock AIO mutex"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Push request aio_queue_.push(aio_request); // Signal AIO thread if(pthread_cond_signal(&aio_cond_)) { std::string errmsg = "Cannot signal AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Unlock AIO mutext if(pthread_mutex_unlock(&aio_mtx_)) { std::string errmsg = "Cannot unlock AIO mutex"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Success return TILEDB_AR_OK; } int Array::aio_thread_create() { // Trivial case if(aio_thread_created_) return TILEDB_AR_OK; // Create the thread that will be handling all AIO requests int rc; if((rc=pthread_create(&aio_thread_, NULL, Array::aio_handler, this))) { std::string errmsg = "Cannot create AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } aio_thread_created_ = true; // Success return TILEDB_AR_OK; } int Array::aio_thread_destroy() { // Trivial case if(!aio_thread_created_) return TILEDB_AR_OK; // Lock AIO mutext if(pthread_mutex_lock(&aio_mtx_)) { std::string errmsg = "Cannot lock AIO mutex while destroying AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Signal the cancelation so that the thread unblocks aio_thread_canceled_ = true; if(pthread_cond_signal(&aio_cond_)) { std::string errmsg = "Cannot signal AIO thread while destroying AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Unlock AIO mutext if(pthread_mutex_unlock(&aio_mtx_)) { std::string errmsg = "Cannot unlock AIO mutex while destroying AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Wait for cancelation to take place while(aio_thread_created_); // Join with the terminated thread if(pthread_join(aio_thread_, NULL)) { std::string errmsg = "Cannot join AIO thread"; PRINT_ERROR(errmsg); tiledb_ar_errmsg = TILEDB_AR_ERRMSG + errmsg; return TILEDB_AR_ERR; } // Success return TILEDB_AR_OK; } std::string Array::new_fragment_name() const { struct timeval tp; gettimeofday(&tp, NULL); uint64_t ms = (uint64_t) tp.tv_sec * 1000L + tp.tv_usec / 1000; pthread_t self = pthread_self(); uint64_t tid = 0; memcpy(&tid, &self, std::min(sizeof(self), sizeof(tid))); char fragment_name[TILEDB_NAME_MAX_LEN]; #ifdef USE_MAC_ADDRESS_IN_FRAGMENT_NAMES // Get MAC address std::string mac = get_mac_addr(); if(mac == "") return ""; #else //Use uuids uuid_t value; uuid_generate(value); char uuid_str[40]; uuid_unparse(value, uuid_str); std::string mac = uuid_str; #endif // Generate fragment name. Note: for cloud based filesystems or if locking_support is // turned off explicitly, fragment names are generated "in-place", there will be no // rename associated with those fragments. int n; if (config()->get_filesystem()->locking_support()) { n = snprintf(fragment_name, TILEDB_NAME_MAX_LEN, "%s/.__%s%" PRIu64"_%" PRIu64, get_array_path_used().c_str(), mac.c_str(), tid, ms); } else { n = snprintf(fragment_name, TILEDB_NAME_MAX_LEN, "%s/__%s%" PRIu64"_%" PRIu64, get_array_path_used().c_str(), mac.c_str(), tid, ms); } // Handle error if(n<0) return ""; // Return return fragment_name; } int Array::open_fragments( const std::vector& fragment_names, const std::vector& book_keeping) { // Sanity check assert(fragment_names.size() == book_keeping.size()); // Create a fragment object for each fragment directory int fragment_num = fragment_names.size(); for(int i=0; iinit(fragment_names[i], book_keeping[i], mode()) != TILEDB_FG_OK) { tiledb_ar_errmsg = tiledb_fg_errmsg; return TILEDB_AR_ERR; } } // Success return TILEDB_AR_OK; } void Array::free_array_schema() { if(array_schema_) delete array_schema_; array_schema_ = NULL; } const std::string& Array::get_array_path_used() const { return array_path_used_; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/array_iterator.cc000066400000000000000000000243241453617025200245450ustar00rootroot00000000000000/** * @file array_iterator.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2019 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the ArrayIterator class. */ #include "array_iterator.h" /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_AIT_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_ait_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ArrayIterator::ArrayIterator() { array_ = NULL; buffers_ = NULL; buffer_sizes_ = NULL; end_ = false; var_attribute_num_ = 0; } ArrayIterator::~ArrayIterator() { if (expression_) { delete expression_; } } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ const std::string& ArrayIterator::array_name() const { return array_->get_array_path_used(); } bool ArrayIterator::end() const { return end_; } int ArrayIterator::get_value( int attribute_id, const void** value, size_t* value_size) const { // Trivial case if(end_) { *value = NULL; *value_size = 0; std::string errmsg = "Cannot get value; Iterator end reached"; PRINT_ERROR(errmsg); tiledb_ait_errmsg = TILEDB_AIT_ERRMSG + errmsg; return TILEDB_AIT_ERR; } // Get the value int buffer_i = buffer_i_[attribute_id]; int64_t pos = pos_[attribute_id]; size_t cell_size = cell_sizes_[attribute_id]; if(cell_size != TILEDB_VAR_SIZE) { // FIXED *value = static_cast(buffers_[buffer_i]) + pos*cell_size; *value_size = cell_size; } else { // VARIABLE size_t offset = static_cast(buffers_[buffer_i])[pos]; *value = static_cast(buffers_[buffer_i+1]) + offset; if(pos < cell_num_[attribute_id] - 1) *value_size = static_cast(buffers_[buffer_i])[pos+1] - offset; else *value_size = buffer_sizes_[buffer_i+1] - offset; } // Success return TILEDB_AIT_OK; } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int ArrayIterator::init( Array* array, void** buffers, size_t* buffer_sizes, const char* filter_expression) { // Initial assignments array_ = array; buffers_ = buffers; buffer_sizes_ = buffer_sizes; end_ = false; var_attribute_num_ = 0; // Initialize next, cell num, cell sizes, buffer_i and var_attribute_num const ArraySchema* array_schema = array_->array_schema(); const std::vector attribute_ids = array_->attribute_ids(); int attribute_id_num = attribute_ids.size(); pos_.resize(attribute_id_num); cell_num_.resize(attribute_id_num); cell_sizes_.resize(attribute_id_num); buffer_i_.resize(attribute_id_num); for(int i=0, buffer_i=0; icell_size(attribute_ids[i]); buffer_i_[i] = buffer_i; buffer_allocated_sizes_.push_back(buffer_sizes[buffer_i]); if(cell_sizes_[i] != TILEDB_VAR_SIZE) { ++buffer_i; } else { buffer_allocated_sizes_.push_back(buffer_sizes[buffer_i+1]); buffer_i += 2; ++var_attribute_num_; } } int rc = TILEDB_AIT_OK; // Set up filter expression if (filter_expression != NULL && strlen(filter_expression) > 0) { expression_ = new Expression(filter_expression); if (expression_->init(attribute_ids, array_schema) == TILEDB_EXPR_ERR) { tiledb_ait_errmsg = tiledb_expr_errmsg; delete expression_; expression_ = NULL; rc = TILEDB_AIT_ERR; } } rc |= reset_subarray(0); // Return return rc; } int ArrayIterator::reset_subarray(const void* subarray) { end_ = false; //Reset pos_, cell_num_, buffer_sizes_ pos_.assign(pos_.size(), 0ull); cell_num_.assign(cell_num_.size(), 0ull); memcpy(buffer_sizes_, &(buffer_allocated_sizes_[0]), buffer_allocated_sizes_.size()*sizeof(size_t)); // Reset subarray if(subarray && array_->reset_subarray(subarray) != TILEDB_AR_OK) { tiledb_ait_errmsg = tiledb_ar_errmsg; return TILEDB_AIT_ERR; } // Begin read by invoking next() if (next() && !end_) { std::string errmsg = "Array iterator initialization failed"; PRINT_ERROR(errmsg); tiledb_ait_errmsg = TILEDB_AIT_ERRMSG + errmsg; return TILEDB_AIT_ERR; } // Return return TILEDB_AIT_OK; } int ArrayIterator::finalize() { // Finalize int rc = array_->finalize(); delete array_; array_ = NULL; // Error if(rc != TILEDB_AR_OK) { tiledb_ait_errmsg = tiledb_ar_errmsg; return TILEDB_AIT_ERR; } // Success return TILEDB_AIT_OK; } int ArrayIterator::next() { // Trivial case if(end_) { std::string errmsg = "Cannot advance iterator; Iterator end reached"; PRINT_ERROR(errmsg); tiledb_ait_errmsg = TILEDB_AIT_ERRMSG + errmsg; return TILEDB_AIT_ERR; } // Advance iterator int evaluated_cell = true; do { std::vector needs_new_read; const std::vector attribute_ids = array_->attribute_ids(); int attribute_id_num = attribute_ids.size(); for(int i=0; i 0) { // Need to copy buffer_sizes_ and restore at the end. // buffer_sizes_ must be set to 0 for array->read() to work correctly, i.e., // do not fetch new data for fields which still have pending data in // buffers_. However, the correct value of buffer_sizes_ for such fields is // required for correct operation of the iterator in subsequent calls std::vector copy_buffer_sizes(attribute_id_num+var_attribute_num_); // Properly set the buffer sizes for(int i=0; iread(buffers_, buffer_sizes_) != TILEDB_AR_OK) { tiledb_ait_errmsg = tiledb_ar_errmsg; return TILEDB_AIT_ERR; } // Check if read went well and update internal state for(int i=0; ioverflow(attribute_ids[needs_new_read[i]])) { end_ = true; return TILEDB_AIT_OK; } // Error if(buffer_sizes_[buffer_i] == 0 && array_->overflow(attribute_ids[needs_new_read[i]])) { std::string errmsg = "Cannot advance iterator; Buffer overflow"; PRINT_ERROR(errmsg); tiledb_ait_errmsg = TILEDB_AIT_ERRMSG + errmsg; return TILEDB_AIT_ERR; } // Update cell num & pos buffer_i = buffer_i_[needs_new_read[i]]; // Cell Num if(cell_sizes_[needs_new_read[i]] == TILEDB_VAR_SIZE) // VARIABLE cell_num_[needs_new_read[i]] = buffer_sizes_[buffer_i] / sizeof(size_t); else // FIXED cell_num_[needs_new_read[i]] = buffer_sizes_[buffer_i] / cell_sizes_[needs_new_read[i]]; // Reset current cell positions in buffer pos_[needs_new_read[i]] = 0; } // Restore buffer sizes for attributes which have pending data for(int i=0, needs_new_read_idx=0; i(needs_new_read_idx) < needs_new_read.size() && i == needs_new_read[needs_new_read_idx]) // buffer_size would have been ++needs_new_read_idx; // set by array->read() else { //restore buffer size from copy buffer_i = buffer_i_[i]; buffer_sizes_[buffer_i] = copy_buffer_sizes[buffer_i]; if(cell_sizes_[i] == TILEDB_VAR_SIZE) buffer_sizes_[buffer_i+1] = copy_buffer_sizes[buffer_i+1]; } } } if (expression_) { evaluated_cell = expression_->evaluate_cell(buffers_, buffer_sizes_, pos_); if (evaluated_cell == TILEDB_EXPR_ERR) { tiledb_ait_errmsg = tiledb_expr_errmsg; return TILEDB_AIT_ERR; } } } while(!evaluated_cell); // Success return TILEDB_AIT_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/array_read_state.cc000066400000000000000000002273651453617025200250410ustar00rootroot00000000000000/** * @file array_read_state.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the ArrayReadState class. */ #include "array_read_state.h" #include "utils.h" #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_ARS_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_ars_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ArrayReadState::ArrayReadState( const Array* array) : array_(array) { // For easy reference array_schema_ = array_->array_schema(); attribute_num_ = array_schema_->attribute_num(); coords_size_ = array_schema_->coords_size(); // Initializations done_ = false; empty_cells_written_.resize(attribute_num_+1); fragment_cell_pos_ranges_vec_pos_.resize(attribute_num_+1); min_bounding_coords_end_ = NULL; read_round_done_.resize(attribute_num_); subarray_tile_coords_ = NULL; subarray_tile_domain_ = NULL; for(int i=0; i fragments = array_->fragments(); fragment_num_ = fragments.size(); fragment_read_states_.resize(fragment_num_); for(int i=0; iread_state(); } ArrayReadState::~ArrayReadState() { if(min_bounding_coords_end_ != NULL) free(min_bounding_coords_end_); if(subarray_tile_coords_ != NULL) free(subarray_tile_coords_); if(subarray_tile_domain_ != NULL) free(subarray_tile_domain_); int fragment_bounding_coords_num = fragment_bounding_coords_.size(); for(int i=0; iattribute_ids().size(); for(int i=0; ireset_overflow(); if(array_schema_->dense()) { // DENSE if(skip_counts) { tiledb_ar_errmsg = "skip counts only handled for sparse arrays"; return TILEDB_ARS_ERR; } return read_dense(buffers, buffer_sizes); } else // SPARSE return read_sparse(buffers, buffer_sizes, skip_counts); } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ void ArrayReadState::clean_up_processed_fragment_cell_pos_ranges() { // Find the minimum overlapping tile position across all attributes const std::vector& attribute_ids = array_->attribute_ids(); int attribute_id_num = attribute_ids.size(); int64_t min_pos = fragment_cell_pos_ranges_vec_pos_[0]; for(int i=1; i int ArrayReadState::compute_fragment_cell_pos_ranges( FragmentCellRanges& fragment_cell_ranges, FragmentCellPosRanges& fragment_cell_pos_ranges) const { // For easy reference int dim_num = array_schema_->dim_num(); int fragment_id; int64_t fragment_cell_ranges_num = fragment_cell_ranges.size(); // Compute fragment cell position ranges for(int64_t i=0; idense()) { // DENSE // Create a new fragment cell position range FragmentCellPosRange fragment_cell_pos_range; fragment_cell_pos_range.first = fragment_cell_ranges[i].first; CellPosRange& cell_pos_range = fragment_cell_pos_range.second; T* cell_range = static_cast(fragment_cell_ranges[i].second); cell_pos_range.first = array_schema_->get_cell_pos(cell_range); cell_pos_range.second = array_schema_->get_cell_pos(&cell_range[dim_num]); // Insert into the result fragment_cell_pos_ranges.push_back(fragment_cell_pos_range); } else { // SPARSE // Create a new fragment cell position range FragmentCellPosRange fragment_cell_pos_range; if(fragment_read_states_[fragment_cell_ranges[i].first.first]-> get_fragment_cell_pos_range_sparse( fragment_cell_ranges[i].first, static_cast(fragment_cell_ranges[i].second), fragment_cell_pos_range) != TILEDB_RS_OK) { // Error for(int j=i; j void ArrayReadState::compute_min_bounding_coords_end() { // For easy reference int dim_num = array_schema_->dim_num(); // Allocate memeory if(min_bounding_coords_end_ == NULL) min_bounding_coords_end_ = malloc(coords_size_); T* min_bounding_coords_end = static_cast(min_bounding_coords_end_); // Compute min bounding coords end bool first = true; for(int i=0; i(fragment_bounding_coords_[i]); if(fragment_bounding_coords != NULL) { if(first) { memcpy( min_bounding_coords_end, &fragment_bounding_coords[dim_num], coords_size_); first = false; } else if(array_schema_->tile_cell_order_cmp( &fragment_bounding_coords[dim_num], min_bounding_coords_end) < 0) { memcpy( min_bounding_coords_end, &fragment_bounding_coords[dim_num], coords_size_); } } } } template int ArrayReadState::compute_unsorted_fragment_cell_ranges_dense( std::vector& unsorted_fragment_cell_ranges) { // Compute cell ranges for all fragments for(int i=0; idone()) { if(fragment_read_states_[i]->dense()) { // DENSE // Get fragment cell ranges FragmentCellRanges fragment_cell_ranges; if(fragment_read_states_[i]->get_fragment_cell_ranges_dense( i, fragment_cell_ranges) != TILEDB_RS_OK) { tiledb_ars_errmsg = tiledb_rs_errmsg; return TILEDB_ARS_ERR; } // Insert fragment cell ranges to the result unsorted_fragment_cell_ranges.push_back(fragment_cell_ranges); } else { // SPARSE FragmentCellRanges fragment_cell_ranges; FragmentCellRanges fragment_cell_ranges_tmp; do { // Get next overlapping tiles fragment_read_states_[i]->get_next_overlapping_tile_sparse( static_cast(subarray_tile_coords_)); // Get fragment cell ranges fragment_cell_ranges_tmp.clear(); if(fragment_read_states_[i]->get_fragment_cell_ranges_sparse( i, fragment_cell_ranges_tmp) != TILEDB_RS_OK) { tiledb_ars_errmsg = tiledb_rs_errmsg; return TILEDB_ARS_ERR; } // Insert fragment cell ranges to temporary ranges fragment_cell_ranges.insert( fragment_cell_ranges.end(), fragment_cell_ranges_tmp.begin(), fragment_cell_ranges_tmp.end()); } while(!fragment_read_states_[i]->done() && fragment_read_states_[i]->mbr_overlaps_tile()); unsorted_fragment_cell_ranges.push_back(fragment_cell_ranges); } } else { // Append an empty list unsorted_fragment_cell_ranges.push_back(FragmentCellRanges()); } } // Check if some dense fragment completely covers the subarray bool subarray_area_covered = false; for(int i=0; idone() && fragment_read_states_[i]->dense() && fragment_read_states_[i]->subarray_area_covered()) { subarray_area_covered = true; break; } } // Add a fragment that accounts for the empty areas of the array if(!subarray_area_covered) unsorted_fragment_cell_ranges.push_back(empty_fragment_cell_ranges()); // Success return TILEDB_ARS_OK; } template int ArrayReadState::compute_unsorted_fragment_cell_ranges_sparse( std::vector& unsorted_fragment_cell_ranges) { // For easy reference int dim_num = array_schema_->dim_num(); T* min_bounding_coords_end = static_cast(min_bounding_coords_end_); // Compute the relevant fragment cell ranges for(int i=0; i(fragment_bounding_coords_[i]); // Compute new fragment cell ranges if(fragment_bounding_coords != NULL && array_schema_->tile_cell_order_cmp( fragment_bounding_coords, min_bounding_coords_end) <= 0) { FragmentCellRanges fragment_cell_ranges; if(fragment_read_states_[i]->get_fragment_cell_ranges_sparse( i, fragment_bounding_coords, min_bounding_coords_end, fragment_cell_ranges) != TILEDB_RS_OK) { tiledb_ars_errmsg = tiledb_rs_errmsg; return TILEDB_ARS_ERR; } //This might be empty if no cells found in fragment for the query subarray //MBR overlap does not guarantee existence of cells in the subarray unsorted_fragment_cell_ranges.push_back(fragment_cell_ranges); // If the end bounding coordinate is not the same as the smallest one, // update the start bounding coordinate to exceed the smallest // end bounding coordinates if(memcmp( &fragment_bounding_coords[dim_num], min_bounding_coords_end, coords_size_)) { // Get the first coordinates AFTER the min bounding coords end bool coords_retrieved; if(fragment_read_states_[i]->get_coords_after( min_bounding_coords_end, fragment_bounding_coords, coords_retrieved) != TILEDB_RS_OK) { tiledb_ars_errmsg = tiledb_rs_errmsg; return TILEDB_ARS_ERR; } // Sanity check for the sparse case assert(coords_retrieved); } } else { // Append an empty list unsorted_fragment_cell_ranges.push_back(FragmentCellRanges()); } } // Success return TILEDB_ARS_OK; } int ArrayReadState::copy_cells( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset) { size_t remaining_skip_count = 0u; return copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); } int ArrayReadState::copy_cells( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count) { // For easy reference int type = array_schema_->type(attribute_id); // Invoke the proper templated function int rc = TILEDB_ARS_OK; if(type == TILEDB_CHAR) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_INT8) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_INT16) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_INT32) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_INT64) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_UINT8) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_UINT16) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_UINT32) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_UINT64) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_FLOAT32) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else if(type == TILEDB_FLOAT64) rc = copy_cells(attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count); else rc = TILEDB_ARS_ERR; // Handle error if(rc != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Success return TILEDB_ARS_OK; } template int ArrayReadState::copy_cells( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count) { // For easy reference int64_t pos = fragment_cell_pos_ranges_vec_pos_[attribute_id]; FragmentCellPosRanges& fragment_cell_pos_ranges = *fragment_cell_pos_ranges_vec_[pos]; int64_t fragment_cell_pos_ranges_num = fragment_cell_pos_ranges.size(); int fragment_id; // Fragment id int64_t tile_pos; // Tile position in the fragment // Sanity check assert(!array_schema_->var_size(attribute_id)); // Copy the cell ranges one by one for(int64_t i=0; i( attribute_id, buffer, buffer_size, buffer_offset, cell_pos_range, remaining_skip_count); if(overflow_[attribute_id]) break; else continue; } // Handle non-empty fragment if(fragment_read_states_[fragment_id]->copy_cells( attribute_id, tile_pos, buffer, buffer_size, buffer_offset, cell_pos_range, remaining_skip_count) != TILEDB_RS_OK) { tiledb_ars_errmsg = tiledb_rs_errmsg; return TILEDB_ARS_ERR; } // Handle overflow if(fragment_read_states_[fragment_id]->overflow(attribute_id)) { overflow_[attribute_id] = true; break; } } // Handle the case the read round is done for this attribute if(!overflow_[attribute_id]) { ++fragment_cell_pos_ranges_vec_pos_[attribute_id]; read_round_done_[attribute_id] = true; } else { read_round_done_[attribute_id] = false; } // Success return TILEDB_ARS_OK; } int ArrayReadState::copy_cells_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset) { size_t remaining_skip_count = 0u; size_t remaining_skip_count_var = 0u; return copy_cells_var( attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); } int ArrayReadState::copy_cells_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var) { // For easy reference int type = array_schema_->type(attribute_id); // Invoke the proper templated function int rc = TILEDB_ARS_OK; if(type == TILEDB_CHAR) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_INT8) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_INT16) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_INT32) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_INT64) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_UINT8) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_UINT16) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_UINT32) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_UINT64) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_FLOAT32) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else if(type == TILEDB_FLOAT64) rc = copy_cells_var(attribute_id, buffer, buffer_size, buffer_offset,remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var); else rc = TILEDB_ARS_ERR; // Handle error if(rc != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Success return TILEDB_ARS_OK; } template int ArrayReadState::copy_cells_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var) { // For easy reference int64_t pos = fragment_cell_pos_ranges_vec_pos_[attribute_id]; FragmentCellPosRanges& fragment_cell_pos_ranges = *fragment_cell_pos_ranges_vec_[pos]; int64_t fragment_cell_pos_ranges_num = fragment_cell_pos_ranges.size(); int fragment_id; // Fragment id int64_t tile_pos; // Tile position in the fragment // Sanity check assert(array_schema_->var_size(attribute_id)); // Copy the cell ranges one by one for(int64_t i=0; i( attribute_id, buffer, buffer_size, buffer_offset, remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var, cell_pos_range); if(overflow_[attribute_id]) break; else continue; } // Handle non-empty fragment if(fragment_read_states_[fragment_id]->copy_cells_var( attribute_id, tile_pos, buffer, buffer_size, buffer_offset, remaining_skip_count, buffer_var, buffer_var_size, buffer_var_offset, remaining_skip_count_var, cell_pos_range) != TILEDB_RS_OK) { tiledb_ars_errmsg = tiledb_rs_errmsg; return TILEDB_ARS_ERR; } // Handle overflow if(fragment_read_states_[fragment_id]->overflow(attribute_id)) { overflow_[attribute_id] = true; break; } } // Handle the case the read round is done for this attribute if(!overflow_[attribute_id]) { ++fragment_cell_pos_ranges_vec_pos_[attribute_id]; read_round_done_[attribute_id] = true; } else { read_round_done_[attribute_id] = false; } // Success return TILEDB_ARS_OK; } template void ArrayReadState::copy_cells_with_empty( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, const CellPosRange& cell_pos_range, size_t& remaining_skip_count) { // For easy reference size_t cell_size = array_schema_->cell_size(attribute_id); char* buffer_c = static_cast(buffer); int cell_val_num = array_schema_->cell_val_num(attribute_id); // Calculate free space in buffer size_t buffer_free_space = buffer_size - buffer_offset; buffer_free_space = (buffer_free_space / cell_size) * cell_size; if(buffer_free_space == 0 && remaining_skip_count == 0u) { // Overflow overflow_[attribute_id] = true; return; } // Sanity check assert(!array_schema_->var_size(attribute_id)); // Calculate number of empty cells to write int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; //If #cells to skip >= cell_num_left_to_copy, no need to copy anything if(static_cast(cell_num_left_to_copy) <= remaining_skip_count) { remaining_skip_count -= cell_num_left_to_copy; empty_cells_written_[attribute_id] = 0; //done with this range return; } //#cells to copy - deduct remaining_skip_count assert(remaining_skip_count < static_cast(cell_num_left_to_copy)); cell_num_left_to_copy -= remaining_skip_count; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; // Copy empty cells to buffer auto empty = get_tiledb_empty_value(); for(int64_t i=0; i void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var, const CellPosRange& cell_pos_range) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; size_t cell_size_var = sizeof(T); char* buffer_c = static_cast(buffer); char* buffer_var_c = static_cast(buffer_var); // Calculate free space in buffer size_t buffer_free_space = buffer_size - buffer_offset; buffer_free_space = (buffer_free_space / cell_size) * cell_size; size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; // TileDB traverses the offsets and data in lock step - can't have different skip values assert(remaining_skip_count == remaining_skip_count_var); // Handle overflow if((buffer_free_space == 0 || buffer_var_free_space == 0) && remaining_skip_count == 0u) { // Overflow overflow_[attribute_id] = true; return; } // Sanity check assert(array_schema_->var_size(attribute_id)); // Calculate cell number to copy int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; //If #cells to skip >= cell_num_left_to_copy, no need to copy anything if(static_cast(cell_num_left_to_copy) <= remaining_skip_count) { remaining_skip_count -= cell_num_left_to_copy; remaining_skip_count_var -= cell_num_left_to_copy; empty_cells_written_[attribute_id] = 0; //done with this range return; } //#cells to copy - deduct remaining_skip_count assert(remaining_skip_count < static_cast(cell_num_left_to_copy)); cell_num_left_to_copy -= remaining_skip_count; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); size_t bytes_to_copy_var = std::min(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; cell_num_to_copy = std::min(cell_num_to_copy, cell_num_to_copy_var); // Copy empty cells to buffers //FIXME: can't empty var cells be represented by setting the same value of //buffer_var_offset for consecutive cells? No empty var char needed? auto empty = get_tiledb_empty_value(); for(int64_t i=0; i ArrayReadState::FragmentCellRanges ArrayReadState::empty_fragment_cell_ranges() const { // For easy reference int dim_num = array_schema_->dim_num(); int cell_order = array_schema_->cell_order(); size_t cell_range_size = 2*coords_size_; const T* subarray = static_cast(array_->subarray()); const T* tile_coords = (const T*) subarray_tile_coords_; // To return FragmentInfo fragment_info = FragmentInfo(-1, -1); FragmentCellRanges fragment_cell_ranges; // Compute the tile subarray T* tile_subarray = new T[2*dim_num]; array_schema_->get_tile_subarray(tile_coords, tile_subarray); // Compute overlap of tile subarray with non-empty fragment domain T* query_tile_overlap_subarray = new T[2*dim_num]; int overlap = array_schema_->subarray_overlap( subarray, tile_subarray, query_tile_overlap_subarray); // Contiguous cells, single cell range if(overlap == 1 || overlap == 3) { void* cell_range = malloc(cell_range_size); T* cell_range_T = static_cast(cell_range); for(int i=0; i(cell_range); for(int i=0; i 0 && coords[i] > query_tile_overlap_subarray[2*i+1]) { coords[i] = query_tile_overlap_subarray[2*i]; ++coords[--i]; } } } else if(cell_order == TILEDB_COL_MAJOR) { // COLUMN while(coords[dim_num-1] <= query_tile_overlap_subarray[2*(dim_num-1)+1]) { // Make a cell range representing a slab void* cell_range = malloc(cell_range_size); T* cell_range_T = static_cast(cell_range); for(int i=dim_num-1; i>0; --i) { cell_range_T[i] = coords[i]; cell_range_T[dim_num+i] = coords[i]; } cell_range_T[0] = query_tile_overlap_subarray[0]; cell_range_T[dim_num] = query_tile_overlap_subarray[1]; // Insert the new range into the result vector fragment_cell_ranges.push_back( FragmentCellRange(fragment_info, cell_range)); // Advance coordinates i=1; ++coords[i]; while(i query_tile_overlap_subarray[2*i+1]) { coords[i] = query_tile_overlap_subarray[2*i]; ++coords[++i]; } } } else { assert(0); } // Clean up delete [] coords; } // Clean up delete [] tile_subarray; delete [] query_tile_overlap_subarray; // Return return fragment_cell_ranges; } template int ArrayReadState::get_next_fragment_cell_ranges_dense() { // Trivial case if(done_) return TILEDB_ARS_OK; // Get the next overlapping tile for each fragment get_next_overlapping_tiles_dense(); // Return if there are no more overlapping tiles if(done_) return TILEDB_ARS_OK; // Compute the unsorted fragment cell ranges needed for this read run std::vector unsorted_fragment_cell_ranges; if(compute_unsorted_fragment_cell_ranges_dense( unsorted_fragment_cell_ranges) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Sort fragment cell ranges FragmentCellRanges fragment_cell_ranges; if(sort_fragment_cell_ranges( unsorted_fragment_cell_ranges, fragment_cell_ranges) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Compute the fragment cell position ranges FragmentCellPosRanges* fragment_cell_pos_ranges = new FragmentCellPosRanges(); if(compute_fragment_cell_pos_ranges( fragment_cell_ranges, *fragment_cell_pos_ranges) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Insert cell pos ranges in the state fragment_cell_pos_ranges_vec_.push_back(fragment_cell_pos_ranges); // Clean up processed overlapping tiles clean_up_processed_fragment_cell_pos_ranges(); // Success return TILEDB_ARS_OK; } template int ArrayReadState::get_next_fragment_cell_ranges_sparse() { // Trivial case if(done_) return TILEDB_ARS_OK; // Gets the next overlapping tiles in the fragment read states get_next_overlapping_tiles_sparse(); // Return if there are no more overlapping tiles if(done_) return TILEDB_ARS_OK; // Compute smallest end bounding coordinates compute_min_bounding_coords_end(); // Compute the unsorted fragment cell ranges needed for this read run std::vector unsorted_fragment_cell_ranges; if(compute_unsorted_fragment_cell_ranges_sparse( unsorted_fragment_cell_ranges) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; //MBR overlap does not guarantee existence of cells in the subarray //compute_unsorted_fragment_cell_ranges_sparse might determine no cells //exist in the subarray bool not_found = true; for(const auto& vec : unsorted_fragment_cell_ranges) { if(!vec.empty()) { not_found = false; break; } } if(not_found) { // TODO: Replace recursion return get_next_fragment_cell_ranges_sparse(); } // Sort fragment cell ranges FragmentCellRanges fragment_cell_ranges; if(sort_fragment_cell_ranges( unsorted_fragment_cell_ranges, fragment_cell_ranges) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Compute the fragment cell position ranges FragmentCellPosRanges* fragment_cell_pos_ranges = new FragmentCellPosRanges(); if(compute_fragment_cell_pos_ranges( fragment_cell_ranges, *fragment_cell_pos_ranges) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Insert cell pos ranges in the state fragment_cell_pos_ranges_vec_.push_back(fragment_cell_pos_ranges); // Clean up processed overlapping tiles clean_up_processed_fragment_cell_pos_ranges(); // Success return TILEDB_ARS_OK; } template void ArrayReadState::get_next_overlapping_tiles_dense() { // For easy reference int dim_num = array_schema_->dim_num(); // Get the first overlapping tile for each fragment if(fragment_cell_pos_ranges_vec_.size() == 0) { // Initialize subarray tile coordinates init_subarray_tile_coords(); // Return if there are no more overlapping tiles if(subarray_tile_coords_ == NULL) { done_ = true; return; } // Get next overlapping tile for(int i=0; idense()) fragment_read_states_[i]->get_next_overlapping_tile_dense( static_cast(subarray_tile_coords_)); // else, it is handled in compute_unsorted_fragment_cell_ranges } } else { // Temporarily store the current subarray tile coordinates assert(subarray_tile_coords_ != NULL); T* previous_subarray_tile_coords = new T[dim_num]; memcpy( previous_subarray_tile_coords, subarray_tile_coords_, coords_size_); // Advance range coordinates get_next_subarray_tile_coords(); // Return if there are no more overlapping tiles if(subarray_tile_coords_ == NULL) { done_ = true; delete [] previous_subarray_tile_coords; return; } // Get next overlapping tiles for the processed fragments for(int i=0; idone()) { if(fragment_read_states_[i]->dense()) fragment_read_states_[i]->get_next_overlapping_tile_dense( static_cast(subarray_tile_coords_)); // else, it is handled in compute_unsorted_fragment_cell_ranges } } // Clean up delete [] previous_subarray_tile_coords; } } template void ArrayReadState::get_next_overlapping_tiles_sparse() { // For easy reference int dim_num = array_schema_->dim_num(); // Get the first overlapping tile for each fragment if(fragment_bounding_coords_.size() == 0) { // Initializations assert(fragment_cell_pos_ranges_vec_.size() == 0); fragment_bounding_coords_.resize(fragment_num_); // Get next overlapping tile and bounding coordinates done_ = true; for(int i=0; iget_next_overlapping_tile_sparse(); if(!fragment_read_states_[i]->done()) { fragment_bounding_coords_[i] = malloc(2*coords_size_); fragment_read_states_[i]->get_bounding_coords( fragment_bounding_coords_[i]); done_ = false; } else { fragment_bounding_coords_[i] = NULL; } } } else { // Get the next overlapping tile for the appropriate fragments for(int i=0; i(fragment_bounding_coords_[i]); if(fragment_bounding_coords_[i] != NULL && !memcmp( // Coinciding end bounding coords &fragment_bounding_coords[dim_num], min_bounding_coords_end_, coords_size_)) { fragment_read_states_[i]->get_next_overlapping_tile_sparse(); if(!fragment_read_states_[i]->done()) { fragment_read_states_[i]->get_bounding_coords( fragment_bounding_coords_[i]); } else { if(fragment_bounding_coords_[i]) free(fragment_bounding_coords_[i]); fragment_bounding_coords_[i] = NULL; } } } // Check if done done_ = true; for(int i=0; i void ArrayReadState::init_subarray_tile_coords() { // For easy reference int dim_num = array_schema_->dim_num(); const T* tile_extents = static_cast(array_schema_->tile_extents()); const T* subarray = static_cast(array_->subarray()); // Sanity checks assert(tile_extents != NULL); assert(subarray_tile_domain_ == NULL); // Allocate space for tile domain and subarray tile domain T* tile_domain = new T[2*dim_num]; subarray_tile_domain_ = malloc(2*dim_num*sizeof(T)); T* subarray_tile_domain = static_cast(subarray_tile_domain_); // Get subarray in tile domain array_schema_->get_subarray_tile_domain( subarray, tile_domain, subarray_tile_domain); // Check if there is any overlap between the subarray tile domain and the // array tile domain bool overlap = true; for(int i=0; i tile_domain[2*i+1] || subarray_tile_domain[2*i+1] < tile_domain[2*i]) { overlap = false; break; } } // Calculate subarray tile coordinates if(!overlap) { // No overlap free(subarray_tile_domain_); subarray_tile_domain_ = NULL; assert(subarray_tile_coords_ == NULL); } else { // Overlap subarray_tile_coords_ = malloc(coords_size_); T* subarray_tile_coords = static_cast(subarray_tile_coords_); for(int i=0; i void ArrayReadState::get_next_subarray_tile_coords() { // For easy reference int dim_num = array_schema_->dim_num(); T* subarray_tile_domain = static_cast(subarray_tile_domain_); T* subarray_tile_coords = static_cast(subarray_tile_coords_); // Advance subarray tile coordinates array_schema_->get_next_tile_coords( subarray_tile_domain, subarray_tile_coords); // Check if the new subarray coordinates fall out of the range domain bool inside_domain = true; for(int i=0; i subarray_tile_domain[2*i+1]) { inside_domain = false; break; } } // The coordinates fall outside the domain if(!inside_domain) { free(subarray_tile_domain_); subarray_tile_domain_ = NULL; free(subarray_tile_coords_); subarray_tile_coords_ = NULL; } } int ArrayReadState::read_dense( void** buffers, size_t* buffer_sizes) { // For easy reference std::vector attribute_ids = array_->attribute_ids(); int attribute_id_num = attribute_ids.size(); // Read each attribute individually int buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) { // FIXED CELLS if(read_dense_attr( attribute_ids[i], buffers[buffer_i], buffer_sizes[buffer_i]) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; ++buffer_i; } else { // VARIABLE-SIZED CELLS if(read_dense_attr_var( attribute_ids[i], buffers[buffer_i], // offsets buffer_sizes[buffer_i], buffers[buffer_i+1], // actual values buffer_sizes[buffer_i+1]) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; buffer_i += 2; } } // Success return TILEDB_ARS_OK; } int ArrayReadState::read_dense_attr( int attribute_id, void* buffer, size_t& buffer_size) { // For easy reference int coords_type = array_schema_->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) { return read_dense_attr( attribute_id, buffer, buffer_size); } else if(coords_type == TILEDB_INT64) { return read_dense_attr( attribute_id, buffer, buffer_size); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); tiledb_ars_errmsg = TILEDB_ARS_ERRMSG + errmsg; return TILEDB_ARS_ERR; } } template int ArrayReadState::read_dense_attr( int attribute_id, void* buffer, size_t& buffer_size) { // Auxiliary variables size_t buffer_offset = 0; // Until read is done or there is a buffer overflow for(;;) { // Continue copying from the previous unfinished read round if(!read_round_done_[attribute_id]) if(copy_cells( attribute_id, buffer, buffer_size, buffer_offset) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; return TILEDB_ARS_OK; } // Prepare the cell ranges for the next read round if(fragment_cell_pos_ranges_vec_pos_[attribute_id] >= int64_t(fragment_cell_pos_ranges_vec_.size())) { // Get next cell ranges if(get_next_fragment_cell_ranges_dense() != TILEDB_ARS_OK) return TILEDB_ARS_ERR; } // Check if read is done if(done_ && fragment_cell_pos_ranges_vec_pos_[attribute_id] == int64_t(fragment_cell_pos_ranges_vec_.size())) { buffer_size = buffer_offset; return TILEDB_ARS_OK; } // Copy cells to buffers if(copy_cells( attribute_id, buffer, buffer_size, buffer_offset) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for buffer overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; return TILEDB_ARS_OK; } } } int ArrayReadState::read_dense_attr_var( int attribute_id, void* buffer, size_t& buffer_size, void* buffer_var, size_t& buffer_var_size) { // For easy reference int coords_type = array_schema_->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) { return read_dense_attr_var( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size); } else if(coords_type == TILEDB_INT64) { return read_dense_attr_var( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); tiledb_ars_errmsg = TILEDB_ARS_ERRMSG + errmsg; return TILEDB_ARS_ERR; } } template int ArrayReadState::read_dense_attr_var( int attribute_id, void* buffer, size_t& buffer_size, void* buffer_var, size_t& buffer_var_size) { // Auxiliary variables size_t buffer_offset = 0; size_t buffer_var_offset = 0; // Until read is done or there is a buffer overflow for(;;) { // Continue copying from the previous unfinished read round if(!read_round_done_[attribute_id]) if(copy_cells_var( attribute_id, buffer, buffer_size, buffer_offset, buffer_var, buffer_var_size, buffer_var_offset) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; buffer_var_size = buffer_var_offset; return TILEDB_ARS_OK; } // Prepare the cell ranges for the next read round if(fragment_cell_pos_ranges_vec_pos_[attribute_id] >= int64_t(fragment_cell_pos_ranges_vec_.size())) { // Get next cell ranges if(get_next_fragment_cell_ranges_dense() != TILEDB_ARS_OK) return TILEDB_ARS_ERR; } // Check if read is done if(done_ && fragment_cell_pos_ranges_vec_pos_[attribute_id] == int64_t(fragment_cell_pos_ranges_vec_.size())) { buffer_size = buffer_offset; buffer_var_size = buffer_var_offset; return TILEDB_ARS_OK; } // Copy cells to buffers if(copy_cells_var( attribute_id, buffer, buffer_size, buffer_offset, buffer_var, buffer_var_size, buffer_var_offset) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for buffer overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; buffer_var_size = buffer_var_offset; return TILEDB_ARS_OK; } } } int ArrayReadState::read_sparse( void** buffers, size_t* buffer_sizes, size_t* skip_counts) { // For easy reference std::vector attribute_ids = array_->attribute_ids(); int attribute_id_num = attribute_ids.size(); // Find the coordinates buffer int coords_buffer_i = -1; int buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) // FIXED CELLS ++buffer_i; else // VARIABLE-SIZED CELLS buffer_i +=2; } size_t zero_skip_count = 0u; // Read coordinates attribute first if(coords_buffer_i != -1) { if(read_sparse_attr( attribute_num_, buffers[coords_buffer_i], buffer_sizes[coords_buffer_i], skip_counts ? skip_counts[coords_buffer_i] : zero_skip_count) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; } // Read each attribute individually buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) { // FIXED CELLS if(read_sparse_attr( attribute_ids[i], buffers[buffer_i], buffer_sizes[buffer_i], skip_counts ? skip_counts[buffer_i] : zero_skip_count) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; ++buffer_i; } else { // VARIABLE-SIZED CELLS if(read_sparse_attr_var( attribute_ids[i], buffers[buffer_i], // offsets buffer_sizes[buffer_i], skip_counts ? skip_counts[buffer_i] : zero_skip_count, buffers[buffer_i+1], // actual values buffer_sizes[buffer_i+1], skip_counts ? skip_counts[buffer_i+1]: zero_skip_count) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; buffer_i += 2; } } // Success return TILEDB_ARS_OK; } int ArrayReadState::read_sparse_attr( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count) { // For easy reference int coords_type = array_schema_->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) { return read_sparse_attr( attribute_id, buffer, buffer_size, skip_count); } else if(coords_type == TILEDB_INT64) { return read_sparse_attr( attribute_id, buffer, buffer_size, skip_count); } else if(coords_type == TILEDB_FLOAT32) { return read_sparse_attr( attribute_id, buffer, buffer_size, skip_count); } else if(coords_type == TILEDB_FLOAT64) { return read_sparse_attr( attribute_id, buffer, buffer_size, skip_count); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); tiledb_ars_errmsg = TILEDB_ARS_ERRMSG + errmsg; return TILEDB_ARS_ERR; } } template int ArrayReadState::read_sparse_attr( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count) { // Auxiliary variables size_t buffer_offset = 0; // Until read is done or there is a buffer overflow for(;;) { // Continue copying from the previous unfinished read round if(!read_round_done_[attribute_id]) if(copy_cells( attribute_id, buffer, buffer_size, buffer_offset, skip_count) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; return TILEDB_ARS_OK; } // TODO: skip_count is ignored here - functionally correct, but there is // further opportunity to optimize // Prepare the cell ranges for the next read round if(fragment_cell_pos_ranges_vec_pos_[attribute_id] >= int64_t(fragment_cell_pos_ranges_vec_.size())) { // Get next cell ranges if(get_next_fragment_cell_ranges_sparse() != TILEDB_ARS_OK) return TILEDB_ARS_ERR; } // Check if read is done if(done_ && fragment_cell_pos_ranges_vec_pos_[attribute_id] == int64_t(fragment_cell_pos_ranges_vec_.size())) { buffer_size = buffer_offset; return TILEDB_ARS_OK; } // Copy cells to buffers if(copy_cells( attribute_id, buffer, buffer_size, buffer_offset, skip_count) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for buffer overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; return TILEDB_ARS_OK; } } } int ArrayReadState::read_sparse_attr_var( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count, void* buffer_var, size_t& buffer_var_size, size_t& skip_count_var) { // For easy reference int coords_type = array_schema_->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) { return read_sparse_attr_var( attribute_id, buffer, buffer_size, skip_count, buffer_var, buffer_var_size, skip_count_var); } else if(coords_type == TILEDB_INT64) { return read_sparse_attr_var( attribute_id, buffer, buffer_size, skip_count, buffer_var, buffer_var_size, skip_count_var); } else if(coords_type == TILEDB_FLOAT32) { return read_sparse_attr_var( attribute_id, buffer, buffer_size, skip_count, buffer_var, buffer_var_size, skip_count_var); } else if(coords_type == TILEDB_FLOAT64) { return read_sparse_attr_var( attribute_id, buffer, buffer_size, skip_count, buffer_var, buffer_var_size, skip_count_var); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); tiledb_ars_errmsg = TILEDB_ARS_ERRMSG + errmsg; return TILEDB_ARS_ERR; } } template int ArrayReadState::read_sparse_attr_var( int attribute_id, void* buffer, size_t& buffer_size, size_t& skip_count, void* buffer_var, size_t& buffer_var_size, size_t& skip_count_var) { // Auxiliary variables size_t buffer_offset = 0; size_t buffer_var_offset = 0; // Until read is done or there is a buffer overflow for(;;) { // Continue copying from the previous unfinished read round if(!read_round_done_[attribute_id]) if(copy_cells_var( attribute_id, buffer, buffer_size, buffer_offset, skip_count, buffer_var, buffer_var_size, buffer_var_offset, skip_count_var) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; buffer_var_size = buffer_var_offset; return TILEDB_ARS_OK; } // TODO: skip_count is ignored here - functionally correct, but there is // further opportunity to optimize // Prepare the cell ranges for the next read round if(fragment_cell_pos_ranges_vec_pos_[attribute_id] >= int64_t(fragment_cell_pos_ranges_vec_.size())) { // Get next overlapping tiles if(get_next_fragment_cell_ranges_sparse() != TILEDB_ARS_OK) return TILEDB_ARS_ERR; } // Check if read is done if(done_ && fragment_cell_pos_ranges_vec_pos_[attribute_id] == int64_t(fragment_cell_pos_ranges_vec_.size())) { buffer_size = buffer_offset; buffer_var_size = buffer_var_offset; return TILEDB_ARS_OK; } // Copy cells to buffers if(copy_cells_var( attribute_id, buffer, buffer_size, buffer_offset, skip_count, buffer_var, buffer_var_size, buffer_var_offset, skip_count_var) != TILEDB_ARS_OK) return TILEDB_ARS_ERR; // Check for buffer overflow if(overflow_[attribute_id]) { buffer_size = buffer_offset; buffer_var_size = buffer_var_offset; return TILEDB_ARS_OK; } } } template int ArrayReadState::sort_fragment_cell_ranges( std::vector& unsorted_fragment_cell_ranges, FragmentCellRanges& fragment_cell_ranges) const { // For easy reference int fragment_num = (int) unsorted_fragment_cell_ranges.size(); // Calculate the number of non-empty unsorted fragment range lists int non_empty = 0; int first_non_empty = -1; for(int i=0; i 0); // Trivial case - single fragment if(fragment_num == 1) { fragment_cell_ranges = unsorted_fragment_cell_ranges[first_non_empty]; unsorted_fragment_cell_ranges.clear(); return TILEDB_ARS_OK; } // For easy reference int dim_num = array_schema_->dim_num(); const T* domain = static_cast(array_schema_->domain()); const T* tile_extents = static_cast(array_schema_->tile_extents()); const T* tile_coords = static_cast(subarray_tile_coords_); int rc = TILEDB_ARS_OK; // Compute tile domain // This is non-NULL only in the dense array case T* tile_domain = NULL; if(tile_coords != NULL) { tile_domain = new T[2*dim_num]; for(int i=0; i* pq_fragment_cell_range; PQFragmentCellRange* popped; PQFragmentCellRange* top; PQFragmentCellRange* trimmed_top; PQFragmentCellRange* extra_popped; PQFragmentCellRange* left; PQFragmentCellRange* unary; FragmentCellRange result; // Populate queue std::priority_queue< PQFragmentCellRange*, std::vector* >, SmallerPQFragmentCellRange > pq(array_schema_); for(int i=0; i( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from(unsorted_fragment_cell_ranges[i][0]); pq.push(pq_fragment_cell_range); ++rid[i]; } } // Start processing the queue while(!pq.empty()) { // Pop the first entry and mark it as popped popped = pq.top(); pq.pop(); // Last range - insert it into the results and get the next range // for that fragment if(pq.empty()) { popped->export_to(result); fragment_cell_ranges.push_back(result); fid = (popped->fragment_id_ != -1) ? popped->fragment_id_ : fragment_num-1; delete popped; if(rid[fid] == rlen[fid]) { break; } else { pq_fragment_cell_range = new PQFragmentCellRange( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from( unsorted_fragment_cell_ranges[fid][rid[fid]]); pq.push(pq_fragment_cell_range); ++rid[fid]; continue; } } // Mark the second entry (now top) as top top = pq.top(); // Dinstinguish two cases if(popped->dense() || popped->unary()) { // DENSE OR UNARY POPPED // Keep on trimming ranges from the queue while(!pq.empty() && popped->must_trim(top)) { // Cut the top range and re-insert, only if there is partial overlap if(top->ends_after(popped)) { // Create the new trimmed top range trimmed_top = new PQFragmentCellRange( array_schema_, &fragment_read_states_); popped->trim(top, trimmed_top, tile_domain); // Discard top free(top->cell_range_); delete top; pq.pop(); if(trimmed_top->cell_range_ != NULL) { // Re-insert the trimmed range in pq pq.push(trimmed_top); } else { // Get the next range from the top fragment fid = (trimmed_top->fragment_id_ != -1) ? trimmed_top->fragment_id_ : fragment_num-1; if(rid[fid] != rlen[fid]) { pq_fragment_cell_range = new PQFragmentCellRange( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from( unsorted_fragment_cell_ranges[fid][rid[fid]]); pq.push(pq_fragment_cell_range); ++rid[fid]; } // Clear trimmed top delete trimmed_top; } } else { // Get the next range from the top fragment fid = (top->fragment_id_ != -1) ? top->fragment_id_ : fragment_num-1; if(rid[fid] != rlen[fid]) { pq_fragment_cell_range = new PQFragmentCellRange( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from( unsorted_fragment_cell_ranges[fid][rid[fid]]); } // Discard top free(top->cell_range_); delete top; pq.pop(); if(rid[fid] != rlen[fid]) { pq.push(pq_fragment_cell_range); ++rid[fid]; } } // Get a new top if(!pq.empty()) top = pq.top(); } // Potentially split the popped range if(!pq.empty() && popped->must_be_split(top)) { // Split the popped range extra_popped = new PQFragmentCellRange( array_schema_, &fragment_read_states_); popped->split(top, extra_popped, tile_domain); // Re-instert the extra popped range into the queue pq.push(extra_popped); } else { // Get the next range from popped fragment fid = (popped->fragment_id_ != -1) ? popped->fragment_id_ : fragment_num-1; if(rid[fid] != rlen[fid]) { pq_fragment_cell_range = new PQFragmentCellRange( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from( unsorted_fragment_cell_ranges[fid][rid[fid]]); pq.push(pq_fragment_cell_range); ++rid[fid]; } } // Insert the final popped range into the results popped->export_to(result); fragment_cell_ranges.push_back(result); delete popped; } else { // SPARSE POPPED // If popped does not overlap with top, insert popped into results if(!pq.empty() && top->begins_after(popped)) { popped->export_to(result); fragment_cell_ranges.push_back(result); // Get the next range from the popped fragment fid = popped->fragment_id_; if(rid[fid] != rlen[fid]) { pq_fragment_cell_range = new PQFragmentCellRange( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from( unsorted_fragment_cell_ranges[fid][rid[fid]]); pq.push(pq_fragment_cell_range); ++rid[fid]; } delete popped; } else { // Create up to 3 more ranges (left, unary, new popped/right) left = new PQFragmentCellRange( array_schema_, &fragment_read_states_); unary = new PQFragmentCellRange( array_schema_, &fragment_read_states_); popped->split_to_3(top, left, unary); // Get the next range from the popped fragment if(unary->cell_range_ == NULL && popped->cell_range_ == NULL) { fid = popped->fragment_id_; if(rid[fid] != rlen[fid]) { pq_fragment_cell_range = new PQFragmentCellRange( array_schema_, &fragment_read_states_); pq_fragment_cell_range->import_from( unsorted_fragment_cell_ranges[fid][rid[fid]]); pq.push(pq_fragment_cell_range); ++rid[fid]; } } // Insert left to results or discard it if(left->cell_range_ != NULL) { left->export_to(result); fragment_cell_ranges.push_back(result); } delete left; // Insert unary to the priority queue if(unary->cell_range_ != NULL) pq.push(unary); else delete unary; // Re-insert new popped (right) range to the priority queue if(popped->cell_range_ != NULL) pq.push(popped); else delete popped; } } } // Clean up unsorted_fragment_cell_ranges.clear(); if(tile_domain != NULL) delete [] tile_domain; delete [] rlen; delete [] rid; // Clean up in case of error if(rc != TILEDB_ARS_OK) { while(!pq.empty()) { free(pq.top()->cell_range_); delete pq.top(); pq.pop(); } for(int64_t i=0; i ArrayReadState::PQFragmentCellRange::PQFragmentCellRange( const ArraySchema* array_schema, const std::vector* fragment_read_states) { array_schema_ = array_schema; fragment_read_states_ = fragment_read_states; cell_range_ = NULL; fragment_id_ = -1; tile_id_l_ = -1; tile_id_r_ = -1; tile_pos_ = -1; coords_size_ = array_schema_->coords_size(); dim_num_ = array_schema_->dim_num(); } template bool ArrayReadState::PQFragmentCellRange::begins_after( const PQFragmentCellRange* fcr) const { return tile_id_l_ > fcr->tile_id_r_ || (tile_id_l_ == fcr->tile_id_r_ && array_schema_->cell_order_cmp( cell_range_, &(fcr->cell_range_[dim_num_])) > 0); } template bool ArrayReadState::PQFragmentCellRange::dense() const { return fragment_id_ == -1 || (*fragment_read_states_)[fragment_id_]->dense(); } template bool ArrayReadState::PQFragmentCellRange::ends_after( const PQFragmentCellRange* fcr) const { return tile_id_r_ > fcr->tile_id_r_ || (tile_id_r_ == fcr->tile_id_r_ && array_schema_->cell_order_cmp( &cell_range_[dim_num_], &fcr->cell_range_[dim_num_]) > 0); } template void ArrayReadState::PQFragmentCellRange::export_to( FragmentCellRange& fragment_cell_range) { // Copy members fragment_cell_range.second = cell_range_; fragment_cell_range.first.first = fragment_id_; fragment_cell_range.first.second = tile_pos_; } template void ArrayReadState::PQFragmentCellRange::import_from( const FragmentCellRange& fragment_cell_range) { // Copy members cell_range_ = static_cast(fragment_cell_range.second); fragment_id_ = fragment_cell_range.first.first; tile_pos_ = fragment_cell_range.first.second; // Compute tile ids tile_id_l_ = array_schema_->tile_id(cell_range_); tile_id_r_ = array_schema_->tile_id(&cell_range_[dim_num_]); } template bool ArrayReadState::PQFragmentCellRange::must_be_split( const PQFragmentCellRange* fcr) const { return fcr->fragment_id_ > fragment_id_ && (fcr->tile_id_l_ < tile_id_r_ || (fcr->tile_id_l_ == tile_id_r_ && array_schema_->cell_order_cmp( fcr->cell_range_, &cell_range_[dim_num_]) <= 0)); } template bool ArrayReadState::PQFragmentCellRange::must_trim( const PQFragmentCellRange* fcr) const { return fcr->fragment_id_ < fragment_id_ && (fcr->tile_id_l_ > tile_id_l_ || (fcr->tile_id_l_ == tile_id_l_ && array_schema_->cell_order_cmp(fcr->cell_range_, cell_range_) >= 0)) && (fcr->tile_id_l_ < tile_id_r_ || (fcr->tile_id_l_ == tile_id_r_ && array_schema_->cell_order_cmp( fcr->cell_range_, &cell_range_[dim_num_]) <= 0)); } template void ArrayReadState::PQFragmentCellRange::split( const PQFragmentCellRange* fcr, PQFragmentCellRange* fcr_new, const T* tile_domain) { // Create the new range fcr_new->fragment_id_ = fragment_id_; fcr_new->tile_pos_ = tile_pos_; fcr_new->cell_range_ = (T*) malloc(2*coords_size_); fcr_new->tile_id_l_ = fcr->tile_id_l_; memcpy( fcr_new->cell_range_, fcr->cell_range_, coords_size_); fcr_new->tile_id_r_ = tile_id_r_; memcpy( &(fcr_new->cell_range_[dim_num_]), &cell_range_[dim_num_], coords_size_); // Trim the calling object range memcpy(&cell_range_[dim_num_], fcr->cell_range_, coords_size_); array_schema_->get_previous_cell_coords( tile_domain, &cell_range_[dim_num_]); tile_id_r_ = array_schema_->tile_id(&cell_range_[dim_num_]); } template void ArrayReadState::PQFragmentCellRange::split_to_3( const PQFragmentCellRange* fcr, PQFragmentCellRange* fcr_left, PQFragmentCellRange* fcr_unary) { // Initialize fcr_left fcr_left->fragment_id_ = fragment_id_; fcr_left->tile_pos_ = tile_pos_; fcr_left->cell_range_ = (T*) malloc(2*coords_size_); fcr_left->tile_id_l_ = tile_id_l_; memcpy(fcr_left->cell_range_, cell_range_, coords_size_); // Get enclosing coordinates bool left_retrieved, right_retrieved, target_exists; int rc = (*fragment_read_states_)[fragment_id_]->template get_enclosing_coords( tile_pos_, // Tile fcr->cell_range_, // Target coords cell_range_, // Start coords &cell_range_[dim_num_], // End coords &fcr_left->cell_range_[dim_num_], // Left coords cell_range_, // Right coords left_retrieved, // Left retrieved right_retrieved, // Right retrieved target_exists); // Target exists assert(rc == TILEDB_RS_OK); // Clean up if necessary if(left_retrieved) { fcr_left->tile_id_r_ = array_schema_->tile_id(&fcr_left->cell_range_[dim_num_]); } else { free(fcr_left->cell_range_); fcr_left->cell_range_ = NULL; } if(right_retrieved) { tile_id_l_ = array_schema_->tile_id(cell_range_); } else { free(cell_range_); cell_range_ = NULL; } // Create unary range if(target_exists) { fcr_unary->fragment_id_ = fragment_id_; fcr_unary->tile_pos_ = tile_pos_; fcr_unary->cell_range_ = (T*) malloc(2*coords_size_); fcr_unary->tile_id_l_ = fcr->tile_id_l_; memcpy(fcr_unary->cell_range_, fcr->cell_range_, coords_size_); fcr_unary->tile_id_r_ = fcr->tile_id_l_; memcpy(&(fcr_unary->cell_range_[dim_num_]), fcr->cell_range_, coords_size_); } else { fcr_unary->cell_range_ = NULL; } } template void ArrayReadState::PQFragmentCellRange::trim( const PQFragmentCellRange* fcr, PQFragmentCellRange* fcr_trimmed, const T* tile_domain) const { // Construct trimmed range fcr_trimmed->fragment_id_ = fcr->fragment_id_; fcr_trimmed->tile_pos_ = fcr->tile_pos_; fcr_trimmed->cell_range_ = (T*) malloc(2*coords_size_); memcpy(fcr_trimmed->cell_range_, &cell_range_[dim_num_], coords_size_); fcr_trimmed->tile_id_l_ = tile_id_r_; memcpy( &(fcr_trimmed->cell_range_[dim_num_]), &(fcr->cell_range_[dim_num_]), coords_size_); fcr_trimmed->tile_id_r_ = fcr->tile_id_r_; // Advance the left endpoint of the trimmed range bool coords_retrieved; if(fcr_trimmed->dense()) { array_schema_->get_next_cell_coords( // fcr is DENSE tile_domain, fcr_trimmed->cell_range_, coords_retrieved); } else { // fcr is SPARSE int rc = (*fragment_read_states_)[fcr->fragment_id_]->get_coords_after( &(cell_range_[dim_num_]), fcr_trimmed->cell_range_, coords_retrieved); assert(rc == TILEDB_RS_OK); } if(!coords_retrieved) { free(fcr_trimmed->cell_range_); fcr_trimmed->cell_range_ = NULL; } } template bool ArrayReadState::PQFragmentCellRange::unary() const { return !memcmp(cell_range_, &cell_range_[dim_num_], coords_size_); } template ArrayReadState::SmallerPQFragmentCellRange::SmallerPQFragmentCellRange() { array_schema_ = NULL; } template ArrayReadState::SmallerPQFragmentCellRange::SmallerPQFragmentCellRange( const ArraySchema* array_schema) { array_schema_ = array_schema; } template bool ArrayReadState::SmallerPQFragmentCellRange::operator () ( PQFragmentCellRange* a, PQFragmentCellRange* b) const { // Sanity check assert(array_schema_ != NULL); // First check the tile ids if(a->tile_id_l_ < b->tile_id_l_) return false; else if(a->tile_id_l_ > b->tile_id_l_) return true; // else, check the coordinates // Get cell ordering information for the first range endpoints int cmp = array_schema_->cell_order_cmp( a->cell_range_, b->cell_range_); if(cmp < 0) { // a's range start precedes b's return false; } else if(cmp > 0) { // b's range start preceded a's return true; } else { // a's and b's range starts match - latest fragment wins if(a->fragment_id_ < b->fragment_id_) { return true; } else if(a->fragment_id_ > b->fragment_id_) { return false; } else { assert(0); // This should never happen (equal coordinates and fragment id) return false; } } } // Explicit template instantiations template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/array_schema.cc000066400000000000000000003102011453617025200241440ustar00rootroot00000000000000/* * @file array_schema.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2019-2020, 2022 Omics Data Automation, Inc. * * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the ArraySchema class. */ #include "array_schema.h" #include "codec.h" #include "tiledb_constants.h" #include "utils.h" #include #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_AS_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif #define TILEDB_ARRAY_SCHEMA_VERSION_MAX 0xFu /** * Initially, there was no schema versioning and the position that had the array size * value in the older schema is the version now. To avoid conflicts, version tags are now * persisted as really large values and start with the following value. **/ #define TILEDB_ARRAY_SCHEMA_VERSION_START 0xFFFFFFFFu #define GET_REAL_TILEDB_SCHEMA_VERSION(value) TILEDB_ARRAY_SCHEMA_VERSION_START - value /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_as_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ArraySchema::ArraySchema(StorageFS *fs) { cell_num_per_tile_ = -1; coords_for_hilbert_ = NULL; domain_ = NULL; hilbert_curve_ = NULL; tile_extents_ = NULL; tile_domain_ = NULL; tile_coords_aux_ = NULL; version_tag_ = TILEDB_ARRAY_SCHEMA_VERSION_START - TILEDB_ARRAY_SCHEMA_VERSION_TAG; fs_ = fs; } ArraySchema::~ArraySchema() { if(coords_for_hilbert_ != NULL) delete [] coords_for_hilbert_; if(domain_ != NULL) free(domain_); if(hilbert_curve_ != NULL) delete hilbert_curve_; if(tile_extents_ != NULL) free(tile_extents_); if(tile_domain_ != NULL) free(tile_domain_); if(tile_coords_aux_ != NULL) free(tile_coords_aux_); } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ const std::string& ArraySchema::array_workspace() const { return array_workspace_; } const std::string& ArraySchema::array_name() const { return array_name_; } void ArraySchema::array_schema_export(ArraySchemaC* array_schema_c) const { // Set array workspace size_t array_workspace_len = array_workspace_.size(); array_schema_c->array_workspace_ = (char*) malloc(array_workspace_len+1); strcpy(array_schema_c->array_workspace_, array_workspace_.c_str()); // Set array name size_t array_name_len = array_name_.size(); array_schema_c->array_name_ = (char*) malloc(array_name_len+1); strcpy(array_schema_c->array_name_, array_name_.c_str()); // Set attributes and number of attributes. array_schema_c->attribute_num_ = attribute_num_; array_schema_c->attributes_ = (char**) malloc(attribute_num_*sizeof(char*)); for(int i=0; iattributes_[i] = (char*) malloc(attribute_len+1); strcpy(array_schema_c->attributes_[i], attributes_[i].c_str()); } // Set dimensions array_schema_c->dim_num_ = dim_num_; array_schema_c->dimensions_ = (char**) malloc(dim_num_*sizeof(char*)); for(int i=0; idimensions_[i] = (char*) malloc(dimension_len+1); strcpy(array_schema_c->dimensions_[i], dimensions_[i].c_str()); } // Set dense array_schema_c->dense_ = dense_; // Set domain size_t coords_size = this->coords_size(); array_schema_c->domain_ = malloc(2*coords_size); memcpy(array_schema_c->domain_, domain_, 2*coords_size); // Set tile extents if(tile_extents_ == NULL) { array_schema_c->tile_extents_ = NULL; } else { array_schema_c->tile_extents_ = malloc(coords_size); memcpy(array_schema_c->tile_extents_, tile_extents_, coords_size); } // Set types array_schema_c->types_ = (int*) malloc((attribute_num_+1)*sizeof(int)); for(int i=0; itypes_[i] = types_[i]; // Set cell val num array_schema_c->cell_val_num_ = (int*) malloc((attribute_num_)*sizeof(int)); for(int i=0; icell_val_num_[i] = cell_val_num_[i]; // Set cell order array_schema_c->cell_order_ = cell_order_; // Set tile order array_schema_c->tile_order_ = tile_order_; // Set capacity array_schema_c->capacity_ = capacity_; // Set compression array_schema_c->compression_ = (int*) malloc((attribute_num_+1)*sizeof(int)); array_schema_c->compression_level_ = (int*) malloc((attribute_num_+1)*sizeof(int)); for(int i=0; icompression_[i] = compression_[i]; array_schema_c->compression_level_[i] = compression_level_[i]; } // Set offsets compression array_schema_c->offsets_compression_ = (int*) malloc((attribute_num_+1)*sizeof(int)); array_schema_c->offsets_compression_level_ = (int*) malloc((attribute_num_+1)*sizeof(int)); for(int i=0; ioffsets_compression_[i] = offsets_compression_[i]; array_schema_c->offsets_compression_level_[i] = offsets_compression_level_[i]; } } void ArraySchema::array_schema_export( MetadataSchemaC* metadata_schema_c) const { // Set metadata workspace size_t array_workspace_len = array_workspace_.size(); metadata_schema_c->metadata_workspace_ = (char*) malloc(array_workspace_len+1); strcpy(metadata_schema_c->metadata_name_, array_workspace_.c_str()); // Set metadata name size_t array_name_len = array_name_.size(); metadata_schema_c->metadata_name_ = (char*) malloc(array_name_len+1); strcpy(metadata_schema_c->metadata_name_, array_name_.c_str()); // Set attributes and number of attributes metadata_schema_c->attribute_num_ = attribute_num_ - 1; metadata_schema_c->attributes_ = (char**) malloc((attribute_num_-1)*sizeof(char*)); for(int i=0; iattributes_[i] = (char*) malloc(attribute_len+1); strcpy(metadata_schema_c->attributes_[i], attributes_[i].c_str()); } // Set types metadata_schema_c->types_ = (int*) malloc((attribute_num_-1)*sizeof(int)); for(int i=0; itypes_[i] = types_[i]; // Set cell val num metadata_schema_c->cell_val_num_ = (int*) malloc((attribute_num_-1)*sizeof(int)); for(int i=0; icell_val_num_[i] = cell_val_num_[i]; // Set capacity metadata_schema_c->capacity_ = capacity_; // Set compression metadata_schema_c->compression_ = (int*) malloc(attribute_num_*sizeof(int)); metadata_schema_c->compression_level_ = (int*) malloc(attribute_num_*sizeof(int)); for(int i=0; icompression_[i] = compression_[i]; metadata_schema_c->compression_[i] = compression_level_[i]; } } const std::string& ArraySchema::attribute(int attribute_id) const { assert(attribute_id >= 0 && attribute_id <= attribute_num_+1); // Special case for the search attribute (same as coordinates) if(attribute_id == attribute_num_+1) attribute_id = attribute_num_; return attributes_[attribute_id]; } int ArraySchema::attribute_id(const std::string& attribute) const { // Special case - coordinates if(attribute == TILEDB_COORDS) return attribute_num_; for(int i=0; i& ArraySchema::attributes() const { return attributes_; } int64_t ArraySchema::capacity() const { return capacity_; } int64_t ArraySchema::cell_num_per_tile() const { // Sanity check assert(dense_); return cell_num_per_tile_; } int ArraySchema::cell_order() const { return cell_order_; } size_t ArraySchema::cell_size(int attribute_id) const { // Special case for the search tile if(attribute_id == attribute_num_+1) attribute_id = attribute_num_; return cell_sizes_[attribute_id]; } int ArraySchema::cell_val_num(int attribute_id) const { return cell_val_num_[attribute_id]; } int ArraySchema::compression(int attribute_id) const { assert(attribute_id >= 0 && attribute_id <= attribute_num_+1); // Special case for the "search tile", which is essentially the // coordinates tile if(attribute_id == attribute_num_+1) attribute_id = attribute_num_; return compression_[attribute_id]; } int ArraySchema::compression_level(int attribute_id) const { assert(attribute_id >= 0 && attribute_id <= attribute_num_+1); // Special case for the "search tile", which is essentially the // coordinates tile if(attribute_id == attribute_num_+1) attribute_id = attribute_num_; return compression_level_[attribute_id]; } int ArraySchema::offsets_compression(int attribute_id) const { assert(attribute_id >= 0 && attribute_id < attribute_num_); return offsets_compression_[attribute_id]; } int ArraySchema::offsets_compression_level(int attribute_id) const { assert(attribute_id >= 0 && attribute_id < attribute_num_+1); return offsets_compression_level_[attribute_id]; } size_t ArraySchema::coords_size() const { return coords_size_; } int ArraySchema::coords_type() const { return types_[attribute_num_]; } bool ArraySchema::dense() const { return dense_; } int ArraySchema::dim_num() const { return dim_num_; } const void* ArraySchema::domain() const { return domain_; } int ArraySchema::get_attribute_ids( const std::vector& attributes, std::vector& attribute_ids) const { // Initialization attribute_ids.clear(); int attribute_num = attributes.size(); int id; // Get attribute ids for(int i=0; i(range)); else if(types_[attribute_num_] == TILEDB_INT64) return is_contained_in_tile_slab_col(static_cast(range)); else if(types_[attribute_num_] == TILEDB_FLOAT32) return is_contained_in_tile_slab_col(static_cast(range)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return is_contained_in_tile_slab_col(static_cast(range)); else return false; } bool ArraySchema::is_contained_in_tile_slab_row(const void* range) const { if(types_[attribute_num_] == TILEDB_INT32) return is_contained_in_tile_slab_row(static_cast(range)); else if(types_[attribute_num_] == TILEDB_INT64) return is_contained_in_tile_slab_row(static_cast(range)); else if(types_[attribute_num_] == TILEDB_FLOAT32) return is_contained_in_tile_slab_row(static_cast(range)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return is_contained_in_tile_slab_row(static_cast(range)); else return false; } static void print_compression_type(int compression) { int compression_type = compression & COMPRESS; if(compression_type == TILEDB_GZIP) std::cout << "\tType = GZIP"; else if(compression_type == TILEDB_ZSTD) std::cout << "\tType = ZSTD"; else if(compression_type == TILEDB_LZ4) std::cout << "\tType = : LZ4"; else if(compression_type == TILEDB_BLOSC) std::cout << "\tType = BLOSC"; else if(compression_type == TILEDB_BLOSC_LZ4) std::cout << "\tType = BLOSC_LZ4"; else if(compression_type == TILEDB_BLOSC_LZ4HC) std::cout << "\tType = BLOSC_LZ4HC"; else if(compression_type == TILEDB_BLOSC_SNAPPY) std::cout << "\tType = BLOSC_SNAPPY"; else if(compression_type == TILEDB_BLOSC_ZLIB) std::cout << "\tType = BLOSC_ZLIB"; else if(compression_type == TILEDB_BLOSC_ZSTD) std::cout << "\tType = BLOSC_ZSTD"; else if(compression_type == TILEDB_RLE) std::cout << "\tType = RLE"; else if(compression_type == TILEDB_NO_COMPRESSION) std::cout << "\tType = NONE"; else std::cout << "\tType =" << std::to_string(compression); int pre_compression_type = compression & PRE_COMPRESS; if (pre_compression_type == TILEDB_DELTA_ENCODE) std::cout << " + DELTA_ENCODE"; else if(pre_compression_type == TILEDB_BIT_SHUFFLE) std::cout << " + BIT_SHUFFLE"; std::cout << "\n"; } void ArraySchema::print() const { // Array version std::cout << "Array Schema Version:\n\t" << std::to_string(get_version()) << "\n"; // Array workspace - this is not supported anymore - present for backward compatibility if (array_workspace_.size() > 0) { std::cout << "Array workspace:\n\t" << array_workspace_ << "\n"; } // Array name std::cout << "Array name:\n\t" << array_name_ << "\n"; // Dimension names std::cout << "Dimension names:\n"; for(int i=0; i(types_[i]); assert(offset + sizeof(char) < buffer_size); memcpy(buffer + offset, &type, sizeof(char)); offset += sizeof(char); } // Copy cell_val_num_ for(int i=0; i int ArraySchema::subarray_overlap( const T* subarray_a, const T* subarray_b, T* overlap_subarray) const { // Get overlap range for(int i=0; i subarray_b[2*i+1] || overlap_subarray[2*i+1] < subarray_b[2*i]) { overlap = 0; break; } } // Check partial overlap if(overlap == 1) { for(int i=0; i=0; --i) { if(overlap_subarray[2*i] != subarray_b[2*i] || overlap_subarray[2*i+1] != subarray_b[2*i+1]) { overlap = 2; break; } } } } // Return return overlap; } const void* ArraySchema::tile_domain() const { return tile_domain_; } const void* ArraySchema::tile_extents() const { return tile_extents_; } int64_t ArraySchema::tile_num() const { // Invoke the proper template function if(types_[attribute_num_] == TILEDB_INT32) return tile_num(); else if(types_[attribute_num_] == TILEDB_INT64) return tile_num(); assert(0); std::string errmsg = "Unsupported dimensions type for retrieving the " "number of tiles"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } template int64_t ArraySchema::tile_num() const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); int64_t ret = 1; for(int i=0; i(static_cast(range)); else if(types_[attribute_num_] == TILEDB_INT64) return tile_num(static_cast(range)); assert(0); std::string errmsg = "Unsupported dimensions type for retrieving the " "number of tiles"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } template int64_t ArraySchema::tile_num(const T* range) const { // For easy reference const T* tile_extents = static_cast(tile_extents_); const T* domain = static_cast(domain_); int64_t ret = 1; int64_t start, end; for(int i=0; i(subarray)); else if(types_[attribute_num_] == TILEDB_INT64) return tile_slab_col_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_FLOAT32) return tile_slab_col_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return tile_slab_col_cell_num(static_cast(subarray)); else return TILEDB_AS_ERR; } int64_t ArraySchema::tile_slab_row_cell_num(const void* subarray) const { // Invoke the proper templated function if(types_[attribute_num_] == TILEDB_INT32) return tile_slab_row_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_INT64) return tile_slab_row_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_FLOAT32) return tile_slab_row_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return tile_slab_row_cell_num(static_cast(subarray)); else return TILEDB_AS_ERR; } int ArraySchema::type(int i) const { if(i<0 || i>attribute_num_+1) { std::string errmsg = "Cannot retrieve type; Invalid attribute id"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } if (i == attribute_num_+1) { // This is a special cased "search tile" that is basically the coordinate tile return types_[i-1]; } else { return types_[i]; } } size_t ArraySchema::type_size(int i) const { assert(i>=0 && i <= attribute_num_+1); if (i == attribute_num_+1) { // This is a special cased "search tile" that is basically the coordinate tile return type_sizes_[i-1]; } else { return type_sizes_[i]; } } int ArraySchema::var_attribute_num() const { int var_attribute_num = 0; for(int i=0; i(array_schema_bin); size_t buffer_size = array_schema_bin_size; size_t offset = 0; //Check if version tag exists //The latest version of TileDB adds a version tag. The older schema doesn't have any. //By doing a quick check of the version tag, previously loaded arrays can be queried by //the newer version assert(offset + sizeof(unsigned) < buffer_size); memcpy(&version_tag_, buffer + offset, sizeof(unsigned)); //The older version of TileDB does NOT have workspace information if(version_tag_exists()) { //Move past version tag offset += sizeof(unsigned); // Load array_workspace_ int array_workspace_size; assert(offset + sizeof(int) < buffer_size); memcpy(&array_workspace_size, buffer + offset, sizeof(int)); offset += sizeof(int); array_workspace_.resize(array_workspace_size); assert(offset + array_workspace_size < buffer_size); memcpy(&array_workspace_[0], buffer + offset, array_workspace_size); offset += array_workspace_size; } // Load array_name_ int array_name_size; assert(offset + sizeof(int) < buffer_size); memcpy(&array_name_size, buffer + offset, sizeof(int)); offset += sizeof(int); array_name_.resize(array_name_size); assert(offset + array_name_size < buffer_size); memcpy(&array_name_[0], buffer + offset, array_name_size); offset += array_name_size; // Load dense_ assert(offset + sizeof(bool) < buffer_size); memcpy(&dense_, buffer + offset, sizeof(bool)); offset += sizeof(bool); // Load tile_order_ char tile_order; assert(offset + sizeof(char) < buffer_size); memcpy(&tile_order, buffer + offset, sizeof(char)); tile_order_ = static_cast(tile_order); offset += sizeof(char); // Load cell_order_ char cell_order; assert(offset + sizeof(char) < buffer_size); memcpy(&cell_order, buffer + offset, sizeof(char)); cell_order_ = static_cast(cell_order); offset += sizeof(char); // Load capacity_ assert(offset + sizeof(int64_t) < buffer_size); memcpy(&capacity_, buffer + offset, sizeof(int64_t)); offset += sizeof(int64_t); // Load attributes_ assert(offset + sizeof(int) < buffer_size); memcpy(&attribute_num_, buffer + offset, sizeof(int)); offset += sizeof(int); attributes_.resize(attribute_num_); int attribute_size; for(int i=0; i(type); type_sizes_[i] = compute_type_size(i); } // Load cell_val_num_ cell_val_num_.resize(attribute_num_); for(int i=0; i(compression)); // Backward Compatibility if (get_version() < 1L) { // Load compression levels when not found in the schema compression_level_.push_back(Codec::get_default_level(compression)); } if (get_version() < 2L && i < attribute_num_) { if (cell_val_num_[i] == TILEDB_VAR_NUM) { // Load offset compression for versions below 2 that do not have them offsets_compression_.push_back(static_cast(compression)); offsets_compression_level_.push_back(Codec::get_default_level(compression)); } else { offsets_compression_.push_back(0); offsets_compression_level_.push_back(0); } } } // Load compression_level_. Support added in array schema version 1L if (get_version() >= 1L) { char compression_level; for(int i=0; i<=attribute_num_; ++i) { assert(offset + sizeof(char) <= buffer_size); memcpy(&compression_level, buffer + offset, sizeof(char)); offset += sizeof(char); compression_level_.push_back(compression_level); } } // Load offsets_compression_. Support added in array schema version 2L if (get_version() >= 2L) { char offsets_compression; for(int i=0; i(offsets_compression)); } } // Load offsets_compression_level_. Support added in array schema version 2L if (get_version() >= 2L) { char offsets_compression_level; for(int i=0; iarray_workspace_); // Set array name set_array_name(array_schema_c->array_name_); // Set attributes if(set_attributes( array_schema_c->attributes_, array_schema_c->attribute_num_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set capacity set_capacity(array_schema_c->capacity_); // Set dimensions if(set_dimensions( array_schema_c->dimensions_, array_schema_c->dim_num_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set compression if(set_compression(array_schema_c->compression_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set compression_level if(set_compression_level(array_schema_c->compression_level_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set dense set_dense(array_schema_c->dense_); // Set number of values per cell set_cell_val_num(array_schema_c->cell_val_num_); // Set types if(set_types(array_schema_c->types_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set offsets compression if(set_offsets_compression(array_schema_c->offsets_compression_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set offsets compression level if(set_offsets_compression_level(array_schema_c->offsets_compression_level_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set tile extents if(set_tile_extents(array_schema_c->tile_extents_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set cell order if(set_cell_order(array_schema_c->cell_order_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set tile order if(set_tile_order(array_schema_c->tile_order_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Set domain if(set_domain(array_schema_c->domain_) != TILEDB_AS_OK) return TILEDB_AS_ERR; // Compute number of cells per tile compute_cell_num_per_tile(); // Compute tile domain compute_tile_domain(); // Compute tile offsets compute_tile_offsets(); // Initialize Hilbert curve init_hilbert_curve(); // Initialize static auxiliary variables if(tile_coords_aux_ != NULL) free(tile_coords_aux_); tile_coords_aux_ = malloc(coords_size_*dim_num_); if (do_print) { print(); } // Success return TILEDB_AS_OK; } int ArraySchema::init(const MetadataSchemaC* metadata_schema_c) { // Create an array schema C struct and populate it ArraySchemaC array_schema_c; array_schema_c.array_workspace_ = metadata_schema_c->metadata_workspace_; array_schema_c.array_name_ = metadata_schema_c->metadata_name_; array_schema_c.capacity_ = metadata_schema_c->capacity_; array_schema_c.cell_order_ = TILEDB_ROW_MAJOR; array_schema_c.tile_order_ = TILEDB_ROW_MAJOR; array_schema_c.tile_extents_ = NULL; array_schema_c.dense_ = 0; // Set attributes char** attributes = (char**) malloc((metadata_schema_c->attribute_num_+1)*sizeof(char*)); size_t attribute_len; for(int i=0; iattribute_num_; ++i) { attribute_len = strlen(metadata_schema_c->attributes_[i]); attributes[i] = (char*) malloc(attribute_len+1); strcpy(attributes[i], metadata_schema_c->attributes_[i]); } attribute_len = strlen(TILEDB_KEY); attributes[metadata_schema_c->attribute_num_] = (char*) malloc(attribute_len+1); strcpy(attributes[metadata_schema_c->attribute_num_],TILEDB_KEY); array_schema_c.attributes_ = attributes; array_schema_c.attribute_num_ = metadata_schema_c->attribute_num_ + 1; // Set dimensions char** dimensions = (char**) malloc(4*sizeof(char*)); size_t dimension_len; dimension_len = strlen(TILEDB_AS_KEY_DIM1_NAME); dimensions[0] = (char*) malloc(dimension_len+1); strcpy(dimensions[0], TILEDB_AS_KEY_DIM1_NAME); dimension_len = strlen(TILEDB_AS_KEY_DIM2_NAME); dimensions[1] = (char*) malloc(dimension_len+1); strcpy(dimensions[1], TILEDB_AS_KEY_DIM2_NAME); dimension_len = strlen(TILEDB_AS_KEY_DIM3_NAME); dimensions[2] = (char*) malloc(dimension_len+1); strcpy(dimensions[2], TILEDB_AS_KEY_DIM3_NAME); array_schema_c.dimensions_ = dimensions; dimension_len = strlen(TILEDB_AS_KEY_DIM4_NAME); dimensions[3] = (char*) malloc(dimension_len+1); strcpy(dimensions[3], TILEDB_AS_KEY_DIM4_NAME); array_schema_c.dimensions_ = dimensions; array_schema_c.dim_num_ = 4; // Set domain int* domain = (int*) malloc(8*sizeof(int)); for(int i=0; i<4; ++i) { domain[2*i] = INT_MIN; domain[2*i+1] = INT_MAX; } array_schema_c.domain_ = domain; // Set types int* types = (int*) malloc((metadata_schema_c->attribute_num_+2)*sizeof(int)); for(int i=0; iattribute_num_; ++i) types[i] = metadata_schema_c->types_[i]; types[metadata_schema_c->attribute_num_] = TILEDB_CHAR; types[metadata_schema_c->attribute_num_+1] = TILEDB_INT32; array_schema_c.types_ = types; // Set cell num val int* cell_val_num = (int*) malloc((metadata_schema_c->attribute_num_+1)*sizeof(int)); if(metadata_schema_c->cell_val_num_ == NULL) { for(int i=0; iattribute_num_; ++i) cell_val_num[i] = 1; } else { for(int i=0; iattribute_num_; ++i) cell_val_num[i] = metadata_schema_c->cell_val_num_[i]; } cell_val_num[metadata_schema_c->attribute_num_] = TILEDB_VAR_NUM; array_schema_c.cell_val_num_ = cell_val_num; // Set compression int* compression = (int*) malloc((metadata_schema_c->attribute_num_+2)*sizeof(int)); int* compression_level = (int*) malloc((metadata_schema_c->attribute_num_+2)*sizeof(int)); if(metadata_schema_c->compression_ == NULL) { for(int i=0; iattribute_num_+1; ++i) compression[i] = TILEDB_NO_COMPRESSION; } else { for(int i=0; iattribute_num_+1; ++i) { compression[i] = metadata_schema_c->compression_[i]; compression_level[i] = metadata_schema_c->compression_level_[i]; } } compression[metadata_schema_c->attribute_num_+1] = TILEDB_NO_COMPRESSION; array_schema_c.compression_ = compression; // Initialize schema through the array schema C struct init(&array_schema_c); // Clean up for(int i=0; i= 0); // Set capacity if(capacity > 0) capacity_ = capacity; else capacity_ = TILEDB_AS_CAPACITY; } void ArraySchema::set_cell_val_num(const int* cell_val_num) { cell_val_num_.clear(); if(cell_val_num == NULL) { for(int i=0; i= (size_t)(attribute_num_) && "set_cell_val_num() should be called before set_offsets_compression"); offsets_compression_.clear(); // Set offsets compression if(offsets_compression == NULL) { // Use the same compression specified for the attribute if none is specified for offsets. for(int i=0; i TILEDB_NO_COMPRESSION && offsets_compression[i] == TILEDB_NO_COMPRESSION) || (compression_[i] == TILEDB_NO_COMPRESSION && offsets_compression[i] > TILEDB_NO_COMPRESSION)) { std::string errmsg = "Unsupported. For a given VAR attribute, both compression and offsets_compression have to either have compression or not\n"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } offsets_compression_.push_back(offsets_compression[i]); } else { offsets_compression_.push_back(TILEDB_NO_COMPRESSION); } } } // Success return TILEDB_AS_OK; } int ArraySchema::set_offsets_compression_level(int* compression_level) { // Set defaults based on codec assert(offsets_compression_.size() == (size_t)(attribute_num_) && "set_offsets_compression() should be called before set_offsets_compression_level"); offsets_compression_level_.clear(); for(int i=0; i domain_int[2*i+1]) { std::string errmsg = "Cannot set domain; Lower domain bound larger than its " "corresponding upper"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } } else if(types_[attribute_num_] == TILEDB_INT64) { int64_t* domain_int64 = (int64_t*) domain_; for(int i=0; i domain_int64[2*i+1]) { std::string errmsg = "Cannot set domain; Lower domain bound larger than its " "corresponding upper"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } } else if(types_[attribute_num_] == TILEDB_FLOAT32) { float* domain_float = (float*) domain_; for(int i=0; i domain_float[2*i+1]) { std::string errmsg = "Cannot set domain; Lower domain bound larger than its " "corresponding upper"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } } else if(types_[attribute_num_] == TILEDB_FLOAT64) { double* domain_double = (double*) domain_; for(int i=0; i domain_double[2*i+1]) { std::string errmsg = "Cannot set domain; Lower domain bound larger than its " "corresponding upper"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } } else { std::string errmsg = "Cannot set domain; Invalid coordinates type"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } // Success return TILEDB_AS_OK; } int ArraySchema::set_tile_extents(const void* tile_extents) { // Dense arrays must have tile extents if(tile_extents == NULL && dense_) { std::string errmsg = "Cannot set tile extents; Dense arrays must have tile extents"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } // Free existing tile extends if(tile_extents_ != NULL) free(tile_extents_); // Set tile extents if(tile_extents == NULL) { tile_extents_ = NULL; } else { size_t tile_extents_size = coords_size(); tile_extents_ = malloc(tile_extents_size); memcpy(tile_extents_, tile_extents, tile_extents_size); } // Success return TILEDB_AS_OK; } int ArraySchema::set_tile_order(int tile_order) { // Set tile order if(tile_order != TILEDB_ROW_MAJOR && tile_order != TILEDB_COL_MAJOR) { std::string errmsg = "Cannot set tile order; Invalid tile order"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } tile_order_ = tile_order; // Success return TILEDB_AS_OK; } int ArraySchema::set_types(const int* types) { // Sanity check if(types == NULL) { std::string errmsg = "Cannot set types; Types not provided"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } // Set attribute types for(int i=0; i int ArraySchema::cell_order_cmp(const T* coords_a, const T* coords_b) const { // Check if they are equal if(memcmp(coords_a, coords_b, coords_size_) == 0) return 0; // Check for precedence if(cell_order_ == TILEDB_COL_MAJOR) { // COLUMN-MAJOR for(int i=dim_num_-1; i>=0; --i) { if(coords_a[i] < coords_b[i]) return -1; else if(coords_a[i] > coords_b[i]) return 1; } } else if(cell_order_ == TILEDB_ROW_MAJOR) { // ROW-MAJOR for(int i=0; i coords_b[i]) return 1; } } else if(cell_order_ == TILEDB_HILBERT) { // HILBERT // Check hilbert ids int64_t id_a = hilbert_id(coords_a); int64_t id_b = hilbert_id(coords_b); if(id_a < id_b) return -1; else if(id_a > id_b) return 1; // Hilbert ids match - check coordinates in row-major order for(int i=0; i coords_b[i]) return 1; } } else { // Invalid cell order assert(0); } // The program should never reach this point assert(0); return 0; } void ArraySchema::expand_domain(void* domain) const { if(types_[attribute_num_] == TILEDB_INT32) expand_domain(static_cast(domain)); else if(types_[attribute_num_] == TILEDB_INT64) expand_domain(static_cast(domain)); } template void ArraySchema::expand_domain(T* domain) const { // Applicable only to regular tiles if(tile_extents_ == NULL) return; const T* tile_extents = static_cast(tile_extents_); const T* array_domain = static_cast(domain_); for(int i=0; i int64_t ArraySchema::get_cell_pos(const T* coords) const { // Applicable only to dense arrays if(!dense_) { std::string errmsg = "Cannot get cell position; Invalid array type"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } // Invoke the proper function based on the cell order if(cell_order_ == TILEDB_ROW_MAJOR) { return get_cell_pos_row(coords); } else if(cell_order_ == TILEDB_COL_MAJOR) { return get_cell_pos_col(coords); } else { std::string errmsg = "Cannot get cell position; Invalid cell order"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } template void ArraySchema::get_next_cell_coords( const T* domain, T* cell_coords, bool& coords_retrieved) const { // Sanity check assert(dense_); // Invoke the proper function based on the tile order if(cell_order_ == TILEDB_ROW_MAJOR) get_next_cell_coords_row(domain, cell_coords, coords_retrieved); else if(cell_order_ == TILEDB_COL_MAJOR) get_next_cell_coords_col(domain, cell_coords, coords_retrieved); else // Sanity check assert(0); } template void ArraySchema::get_next_tile_coords( const T* domain, T* tile_coords) const { // Sanity check assert(dense_); // Invoke the proper function based on the tile order if(tile_order_ == TILEDB_ROW_MAJOR) get_next_tile_coords_row(domain, tile_coords); else if(tile_order_ == TILEDB_COL_MAJOR) get_next_tile_coords_col(domain, tile_coords); else // Sanity check assert(0); } template void ArraySchema::get_previous_cell_coords( const T* domain, T* cell_coords) const { // Sanity check assert(dense_); // Invoke the proper function based on the tile order if(cell_order_ == TILEDB_ROW_MAJOR) get_previous_cell_coords_row(domain, cell_coords); else if(cell_order_ == TILEDB_COL_MAJOR) get_previous_cell_coords_col(domain, cell_coords); else // Sanity check assert(0); } template void ArraySchema::get_subarray_tile_domain( const T* subarray, T* tile_domain, T* subarray_tile_domain) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); // Get tile domain T tile_num; // Per dimension for(int i=0; i int64_t ArraySchema::get_tile_pos(const T* tile_coords) const { // Sanity check assert(tile_extents_); // Invoke the proper function based on the tile order if(tile_order_ == TILEDB_ROW_MAJOR) { return get_tile_pos_row(tile_coords); } else if(tile_order_ == TILEDB_COL_MAJOR) { return get_tile_pos_col(tile_coords); } else { // Sanity check assert(0); std::string errmsg = "Cannot get tile position; Invalid tile order"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } template int64_t ArraySchema::get_tile_pos( const T* domain, const T* tile_coords) const { // Sanity check assert(tile_extents_); // Invoke the proper function based on the tile order if(tile_order_ == TILEDB_ROW_MAJOR) { return get_tile_pos_row(domain, tile_coords); } else if(tile_order_ == TILEDB_COL_MAJOR) { return get_tile_pos_col(domain, tile_coords); } else { // Sanity check assert(0); std::string errmsg = "Cannot get tile position; Invalid tile order"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; return TILEDB_AS_ERR; } } template void ArraySchema::get_tile_subarray( const T* tile_coords, T* tile_subarray) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); for(int i=0; i int64_t ArraySchema::hilbert_id(const T* coords) const { // For easy reference const T* domain = static_cast(domain_); // Normalize coordinates for(int i = 0; i < dim_num_; ++i) coords_for_hilbert_[i] = static_cast(coords[i] - domain[2*i]); // Compute Hilber id int64_t id; hilbert_curve_->coords_to_hilbert(coords_for_hilbert_, id); // Return return id; } template int ArraySchema::tile_cell_order_cmp( const T* coords_a, const T* coords_b) const { // Check tile order int tile_cmp = tile_order_cmp(coords_a, coords_b); if(tile_cmp) return tile_cmp; // Check cell order return cell_order_cmp(coords_a, coords_b); } template inline int64_t ArraySchema::tile_id(const T* cell_coords) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); // Trivial case if(tile_extents == NULL) return 0; // Calculate tile coordinates T* tile_coords = static_cast(tile_coords_aux_); for(int i=0; i int ArraySchema::tile_order_cmp( const int* coords_a, const int* coords_b) const { // For easy reference int diff; int norm; const int* domain = static_cast(domain_); const int* tile_extents = static_cast(tile_extents_); // If there are regular tiles, first check tile order if(tile_extents_ != NULL) { // ROW-MAJOR if(tile_order_ == TILEDB_ROW_MAJOR) { // Check if the cells are definitely IN the same tile for(int i=0; i 0) norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } else { // COLUMN-MAJOR // Check if the cells are definitely IN the same tile for(int i=dim_num_-1; i>=0; --i) { diff = coords_a[i] - coords_b[i]; if(diff < 0) norm = (coords_a[i] - domain[2*i]) % tile_extents[i]; else if(diff > 0) norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } } // Same tile order return 0; } template<> int ArraySchema::tile_order_cmp( const int64_t* coords_a, const int64_t* coords_b) const { // For easy reference int64_t diff = 0; int64_t norm = 0; const int64_t* domain = static_cast(domain_); const int64_t* tile_extents = static_cast(tile_extents_); // If there are regular tiles, first check tile order if(tile_extents_ != NULL) { // ROW-MAJOR if(tile_order_ == TILEDB_ROW_MAJOR) { // Check if the cells are definitely IN the same tile for(int i=0; i 0) norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } else { // COLUMN-MAJOR // Check if the cells are definitely IN the same tile for(int i=dim_num_-1; i>=0; --i) { diff = coords_a[i] - coords_b[i]; if(diff < 0) norm = (coords_a[i] - domain[2*i]) % tile_extents[i]; else if(diff > 0) norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } } // Same tile order return 0; } template<> int ArraySchema::tile_order_cmp( const float* coords_a, const float* coords_b) const { // For easy reference float diff = 0.0; float norm = 0.0, norm_temp = 0.0; const float* domain = static_cast(domain_); const float* tile_extents = static_cast(tile_extents_); // If there are regular tiles, first check tile order if(tile_extents_ != NULL) { // ROW-MAJOR if(tile_order_ == TILEDB_ROW_MAJOR) { // Check if the cells are definitely IN the same tile for(int i=0; i= domain[2*i]); } else if(diff > 0) { norm_temp = coords_b[i]; do { norm = norm_temp; norm_temp -= tile_extents[i]; } while(norm_temp >= domain[2*i]); } if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } else { // COLUMN-MAJOR // Check if the cells are definitely IN the same tile for(int i=dim_num_-1; i>=0; --i) { diff = coords_a[i] - coords_b[i]; if(diff < 0) { norm_temp = coords_a[i]; do { norm = norm_temp; norm_temp -= tile_extents[i]; } while(norm_temp >= domain[2*i]); } else if(diff > 0) { norm_temp = coords_b[i]; do { norm = norm_temp; norm_temp -= tile_extents[i]; } while(norm_temp >= domain[2*i]); } if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } } // Same tile order return 0; } template<> int ArraySchema::tile_order_cmp( const double* coords_a, const double* coords_b) const { // For easy reference double diff; double norm, norm_temp; const double* domain = static_cast(domain_); const double* tile_extents = static_cast(tile_extents_); // If there are regular tiles, first check tile order if(tile_extents_ != NULL) { // ROW-MAJOR if(tile_order_ == TILEDB_ROW_MAJOR) { // Check if the cells are definitely IN the same tile for(int i=0; i= domain[2*i]); } else if(diff > 0) { norm_temp = coords_b[i]; do { norm = norm_temp; norm_temp -= tile_extents[i]; } while(norm_temp >= domain[2*i]); } if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } else { // COLUMN-MAJOR // Check if the cells are definitely IN the same tile for(int i=dim_num_-1; i>=0; --i) { diff = coords_a[i] - coords_b[i]; if(diff < 0) { norm_temp = coords_a[i]; do { norm = norm_temp; norm_temp -= tile_extents[i]; } while(norm_temp >= domain[2*i]); } else if(diff > 0) { norm_temp = coords_b[i]; do { norm = norm_temp; norm_temp -= tile_extents[i]; } while(norm_temp >= domain[2*i]); } if(diff < 0 && (norm - diff) >= tile_extents[i]) return -1; else if(diff > 0 && (norm + diff) >= tile_extents[i]) return 1; } } } // Same tile order return 0; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ // ===== FORMAT ===== // array_name_size(int) // array_name(string) // dense(bool) // tile_order(char) // cell_order(char) // capacity(int64_t) // attribute_num(int) // attribute_size#1(int) attribute#1(string) // attribute_size#2(int) attribute#2(string) // ... // dim_num(int) // dimension_size#1(int) dimension#1(string) // dimension_size#2(int) dimension#2(string) // ... // domain_size(int) // domain#1_low(coordinates type) dim_domain#1_high(coordinates type) // domain#2_low(coordinates type) dim_domain#2_high(coordinates type) // ... // tile_extents_size(int) // tile_extent#1(coordinates type) tile_extent#2(coordinates type) ... // type#1(char) type#2(char) ... // cell_val_num#1(int) cell_val_num#2(int) ... // compression#1(char) compression#2(char) ... size_t ArraySchema::compute_bin_size() const { // Initialization size_t bin_size = 0; //Size for version tag bin_size += sizeof(unsigned); // Size for array_workspace_ bin_size += sizeof(int) + array_workspace_.size(); // Size for array_name_ bin_size += sizeof(int) + array_name_.size(); // Size for dense_ bin_size += sizeof(bool); // Size for tile_order_ and cell_order_ bin_size += 2 * sizeof(char); // Size for capacity_ bin_size += sizeof(int64_t); // Size for attribute_num_ and attributes_ bin_size += sizeof(int); for(int i=0; i(); else if(coords_type == TILEDB_INT64) compute_cell_num_per_tile(); else // Sanity check assert(0); } template void ArraySchema::compute_cell_num_per_tile() { const T* tile_extents = static_cast(tile_extents_); cell_num_per_tile_ = 1; for(int i=0; i= 0 && i <= attribute_num_); // Variable-sized cell if(i void ArraySchema::compute_hilbert_bits() { // For easy reference const T* domain = static_cast(domain_); T max_domain_range = 0; T domain_range; for(int i = 0; i < dim_num_; ++i) { domain_range = domain[2*i+1] - domain[2*i] + 1; if(max_domain_range < domain_range) max_domain_range = domain_range; } hilbert_bits_ = ceil(log2(int64_t(max_domain_range+0.5))); } void ArraySchema::compute_tile_domain() { // For easy reference int coords_type = types_[attribute_num_]; // Invoke the proper templated function if(coords_type == TILEDB_INT32) compute_tile_domain(); else if(coords_type == TILEDB_INT64) compute_tile_domain(); else if(coords_type == TILEDB_FLOAT32) compute_tile_domain(); else if(coords_type == TILEDB_FLOAT64) compute_tile_domain(); } template void ArraySchema::compute_tile_domain() { if(tile_extents_ == NULL) return; // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); // Allocate space for the tile domain assert(tile_domain_ == NULL); tile_domain_ = malloc(2*dim_num_*sizeof(T)); // For easy reference T* tile_domain = static_cast(tile_domain_); T tile_num; // Per dimension // Calculate tile domain for(int i=0; i(); } else if(types_[attribute_num_] == TILEDB_INT64) { compute_tile_offsets(); } else if(types_[attribute_num_] == TILEDB_FLOAT32) { compute_tile_offsets(); } else if(types_[attribute_num_] == TILEDB_FLOAT64) { compute_tile_offsets(); } else { // The program should never reach this point assert(0); } } template void ArraySchema::compute_tile_offsets() { // Applicable only to non-NULL space tiles if(tile_extents_ == NULL) return; // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); int64_t tile_num; // Per dimension // Calculate tile offsets for column-major tile order tile_offsets_col_.push_back(1); for(int i=1; i=0; --i) { tile_num = (domain[2*(i+1)+1] - domain[2*(i+1)] + 1) / tile_extents[i+1]; tile_offsets_row_.push_back(tile_offsets_row_.back() * tile_num); } std::reverse(tile_offsets_row_.begin(), tile_offsets_row_.end()); } size_t ArraySchema::compute_type_size(int i) const { // Sanity check assert(i>= 0 && i <= attribute_num_); if(types_[i] == TILEDB_CHAR) { return sizeof(char); } else if(types_[i] == TILEDB_INT8) { return sizeof(int8_t); } else if(types_[i] == TILEDB_INT16) { return sizeof(int16_t); } else if(types_[i] == TILEDB_INT32) { return sizeof(int32_t); } else if(types_[i] == TILEDB_INT64) { return sizeof(int64_t); } else if(types_[i] == TILEDB_UINT8) { return sizeof(uint8_t); } else if(types_[i] == TILEDB_UINT16) { return sizeof(uint16_t); } else if(types_[i] == TILEDB_UINT32) { return sizeof(uint32_t); } else if(types_[i] == TILEDB_UINT64) { return sizeof(uint64_t); } else if(types_[i] == TILEDB_FLOAT32) { return sizeof(float); } else if(types_[i] == TILEDB_FLOAT64) { return sizeof(double); } else { // The program should never reach this point assert(0); return 0; } } template int64_t ArraySchema::get_cell_pos_col(const T* coords) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); // Calculate cell offsets int64_t cell_num; // Per dimension std::vector cell_offsets; cell_offsets.push_back(1); for(int i=1; i int64_t ArraySchema::get_cell_pos_row(const T* coords) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); // Calculate cell offsets int64_t cell_num; // Per dimension std::vector cell_offsets; cell_offsets.push_back(1); for(int i=dim_num_-2; i>=0; --i) { cell_num = tile_extents[i+1]; cell_offsets.push_back(cell_offsets.back() * cell_num); } std::reverse(cell_offsets.begin(), cell_offsets.end()); // Calculate position T coords_norm; // Normalized coordinates inside the tile int64_t pos = 0; for(int i=0; i void ArraySchema::get_next_cell_coords_col( const T* domain, T* cell_coords, bool& coords_retrieved) const { int i = 0; ++cell_coords[i]; while(i < dim_num_-1 && cell_coords[i] > domain[2*i+1]) { cell_coords[i] = domain[2*i]; ++cell_coords[++i]; } if(i == dim_num_-1 && cell_coords[i] > domain[2*i+1]) coords_retrieved = false; else coords_retrieved = true; } template void ArraySchema::get_next_cell_coords_row( const T* domain, T* cell_coords, bool& coords_retrieved) const { int i = dim_num_-1; ++cell_coords[i]; while(i > 0 && cell_coords[i] > domain[2*i+1]) { cell_coords[i] = domain[2*i]; ++cell_coords[--i]; } if(i == 0 && cell_coords[i] > domain[2*i+1]) coords_retrieved = false; else coords_retrieved = true; } template void ArraySchema::get_previous_cell_coords_col( const T* domain, T* cell_coords) const { int i = 0; --cell_coords[i]; while(i < dim_num_-1 && cell_coords[i] < domain[2*i]) { cell_coords[i] = domain[2*i+1]; --cell_coords[++i]; } } template void ArraySchema::get_previous_cell_coords_row( const T* domain, T* cell_coords) const { int i = dim_num_-1; --cell_coords[i]; while(i > 0 && cell_coords[i] < domain[2*i]) { cell_coords[i] = domain[2*i+1]; --cell_coords[--i]; } } template void ArraySchema::get_next_tile_coords_col( const T* domain, T* tile_coords) const { int i = 0; ++tile_coords[i]; while(i < dim_num_-1 && tile_coords[i] > domain[2*i+1]) { tile_coords[i] = domain[2*i]; ++tile_coords[++i]; } } template void ArraySchema::get_next_tile_coords_row( const T* domain, T* tile_coords) const { int i = dim_num_-1; ++tile_coords[i]; while(i > 0 && tile_coords[i] > domain[2*i+1]) { tile_coords[i] = domain[2*i]; ++tile_coords[--i]; } } template int64_t ArraySchema::get_tile_pos_col(const T* tile_coords) const { // Calculate position int64_t pos = 0; for(int i=0; i int64_t ArraySchema::get_tile_pos_col( const T* domain, const T* tile_coords) const { // For easy reference const T* tile_extents = static_cast(tile_extents_); // Calculate tile offsets int64_t tile_num; // Per dimension std::vector tile_offsets; tile_offsets.push_back(1); for(int i=1; i int64_t ArraySchema::get_tile_pos_row(const T* tile_coords) const { // Calculate position int64_t pos = 0; for(int i=0; i int64_t ArraySchema::get_tile_pos_row( const T* domain, const T* tile_coords) const { // For easy reference const T* tile_extents = static_cast(tile_extents_); // Calculate tile offsets int64_t tile_num; // Per dimension std::vector tile_offsets; tile_offsets.push_back(1); for(int i=dim_num_-2; i>=0; --i) { tile_num = (domain[2*(i+1)+1] - domain[2*(i+1)] + 1) / tile_extents[i+1]; tile_offsets.push_back(tile_offsets.back() * tile_num); } std::reverse(tile_offsets.begin(), tile_offsets.end()); // Calculate position int64_t pos = 0; for(int i=0; i(); else if(types_[attribute_num_] == TILEDB_INT64) compute_hilbert_bits(); else if(types_[attribute_num_] == TILEDB_FLOAT32) compute_hilbert_bits(); else if(types_[attribute_num_] == TILEDB_FLOAT64) compute_hilbert_bits(); // Create new Hilberrt curve hilbert_curve_ = new HilbertCurve(hilbert_bits_, dim_num_); } template bool ArraySchema::is_contained_in_tile_slab_col(const T* range) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); int64_t tile_l, tile_h; // Check if range is not contained in a column tile slab for(int i=1; i bool ArraySchema::is_contained_in_tile_slab_row(const T* range) const { // For easy reference const T* domain = static_cast(domain_); const T* tile_extents = static_cast(tile_extents_); int64_t tile_l, tile_h; // Check if range is not contained in a row tile slab for(int i=0; i int64_t ArraySchema::tile_slab_col_cell_num(const T* subarray) const { // For easy reference const T* tile_extents = static_cast(tile_extents_); // Initialize the cell num to be returned to the maximum number of rows // in the slab int64_t cell_num = std::min( tile_extents[dim_num_-1], subarray[2*(dim_num_-1)+1] - subarray[2*(dim_num_-1)] + 1); // Calculate the number of cells in the slab for(int i=0; i int64_t ArraySchema::tile_slab_row_cell_num(const T* subarray) const { // For easy reference const T* tile_extents = static_cast(tile_extents_); // Initialize the cell num to be returned to the maximum number of rows // in the slab int64_t cell_num = std::min(tile_extents[0], subarray[1] - subarray[0] + 1); // Calculate the number of cells in the slab for(int i=1; i( const int* coords_a, const int* coords_b) const; template int ArraySchema::cell_order_cmp( const int64_t* coords_a, const int64_t* coords_b) const; template int ArraySchema::cell_order_cmp( const float* coords_a, const float* coords_b) const; template int ArraySchema::cell_order_cmp( const double* coords_a, const double* coords_b) const; template int64_t ArraySchema::get_cell_pos( const int* coords) const; template int64_t ArraySchema::get_cell_pos( const int64_t* coords) const; template int64_t ArraySchema::get_cell_pos( const float* coords) const; template int64_t ArraySchema::get_cell_pos( const double* coords) const; template void ArraySchema::get_next_cell_coords( const int* domain, int* cell_coords, bool& coords_retrieved) const; template void ArraySchema::get_next_cell_coords( const int64_t* domain, int64_t* cell_coords, bool& coords_retrieved) const; template void ArraySchema::get_next_cell_coords( const float* domain, float* cell_coords, bool& coords_retrieved) const; template void ArraySchema::get_next_cell_coords( const double* domain, double* cell_coords, bool& coords_retrieved) const; template void ArraySchema::get_next_tile_coords( const int* domain, int* tile_coords) const; template void ArraySchema::get_next_tile_coords( const int64_t* domain, int64_t* tile_coords) const; template void ArraySchema::get_next_tile_coords( const float* domain, float* tile_coords) const; template void ArraySchema::get_next_tile_coords( const double* domain, double* tile_coords) const; template void ArraySchema::get_previous_cell_coords( const int* domain, int* cell_coords) const; template void ArraySchema::get_previous_cell_coords( const int64_t* domain, int64_t* cell_coords) const; template void ArraySchema::get_previous_cell_coords( const float* domain, float* cell_coords) const; template void ArraySchema::get_previous_cell_coords( const double* domain, double* cell_coords) const; template void ArraySchema::get_subarray_tile_domain( const int* subarray, int* tile_domain, int* subarray_tile_domain) const; template void ArraySchema::get_subarray_tile_domain( const int64_t* subarray, int64_t* tile_domain, int64_t* subarray_tile_domain) const; template int64_t ArraySchema::get_tile_pos( const int* domain, const int* tile_coords) const; template int64_t ArraySchema::get_tile_pos( const int64_t* domain, const int64_t* tile_coords) const; template int64_t ArraySchema::get_tile_pos( const float* domain, const float* tile_coords) const; template int64_t ArraySchema::get_tile_pos( const double* domain, const double* tile_coords) const; template void ArraySchema::get_tile_subarray( const int* tile_coords, int* tile_subarray) const; template void ArraySchema::get_tile_subarray( const int64_t* tile_coords, int64_t* tile_subarray) const; template int64_t ArraySchema::hilbert_id( const int* coords) const; template int64_t ArraySchema::hilbert_id( const int64_t* coords) const; template int64_t ArraySchema::hilbert_id( const float* coords) const; template int64_t ArraySchema::hilbert_id( const double* coords) const; template bool ArraySchema::is_contained_in_tile_slab_col( const int* range) const; template bool ArraySchema::is_contained_in_tile_slab_col( const int64_t* range) const; template bool ArraySchema::is_contained_in_tile_slab_col( const float* range) const; template bool ArraySchema::is_contained_in_tile_slab_col( const double* range) const; template bool ArraySchema::is_contained_in_tile_slab_row( const int* range) const; template bool ArraySchema::is_contained_in_tile_slab_row( const int64_t* range) const; template bool ArraySchema::is_contained_in_tile_slab_row( const float* range) const; template bool ArraySchema::is_contained_in_tile_slab_row( const double* range) const; template int ArraySchema::subarray_overlap( const int* subarray_a, const int* subarray_b, int* overlap_subarray) const; template int ArraySchema::subarray_overlap( const int64_t* subarray_a, const int64_t* subarray_b, int64_t* overlap_subarray) const; template int ArraySchema::subarray_overlap( const float* subarray_a, const float* subarray_b, float* overlap_subarray) const; template int ArraySchema::subarray_overlap( const double* subarray_a, const double* subarray_b, double* overlap_subarray) const; template int ArraySchema::tile_cell_order_cmp( const int* coords_a, const int* coords_b) const; template int ArraySchema::tile_cell_order_cmp( const int64_t* coords_a, const int64_t* coords_b) const; template int ArraySchema::tile_cell_order_cmp( const float* coords_a, const float* coords_b) const; template int ArraySchema::tile_cell_order_cmp( const double* coords_a, const double* coords_b) const; template int64_t ArraySchema::tile_id( const int* cell_coords) const; template int64_t ArraySchema::tile_id( const int64_t* cell_coords) const; template int64_t ArraySchema::tile_id( const float* cell_coords) const; template int64_t ArraySchema::tile_id( const double* cell_coords) const; genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/array_sorted_read_state.cc000066400000000000000000002602631453617025200264130ustar00rootroot00000000000000/** * @file array_sorted_read_state.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the ArraySortedReadState class. */ #include "array_sorted_read_state.h" #include "comparators.h" #include "math.h" #include "utils.h" #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_ASRS_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif #if defined HAVE_OPENMP && defined USE_PARALLEL_SORT #include #define SORT(first, last, comp) __gnu_parallel::sort((first), (last), (comp)) #else #include #define SORT(first, last, comp) std::sort((first), (last), (comp)) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_asrs_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ArraySortedReadState::ArraySortedReadState( Array* array) : array_(array) { // Calculate the attribute ids calculate_attribute_ids(); // For easy reference const ArraySchema* array_schema = array_->array_schema(); int anum = (int) attribute_ids_.size(); // Initializations aio_id_ = 0; aio_cnt_ = 0; coords_size_ = array_schema->coords_size(); copy_id_ = 0; dim_num_ = array_schema->dim_num(); copy_thread_running_ = false; copy_thread_canceled_ = false; read_tile_slabs_done_ = false; resume_copy_ = false; resume_aio_ = false; tile_coords_ = NULL; tile_domain_ = NULL; for(int i=0; i<2; ++i) { aio_overflow_[i] = new bool[anum]; buffer_sizes_[i] = NULL; buffer_sizes_tmp_[i] = NULL; buffer_sizes_tmp_bak_[i] = NULL; buffers_[i] = NULL; tile_slab_[i] = malloc(2*coords_size_); tile_slab_norm_[i] = malloc(2*coords_size_); tile_slab_init_[i] = false; wait_copy_[i] = false; wait_aio_[i] = true; } overflow_ = new bool[anum]; overflow_still_ = new bool[anum]; for(int i=0; ivar_size(attribute_ids_[i])) attribute_sizes_.push_back(sizeof(size_t)); else attribute_sizes_.push_back(array_schema->cell_size(attribute_ids_[i])); } subarray_ = malloc(2*coords_size_); memcpy(subarray_, array_->subarray(), 2*coords_size_); // Calculate number of buffers calculate_buffer_num(); // Calculate buffer sizes calculate_buffer_sizes(); // Initialize tile slab info and state, and copy state init_tile_slab_info(); init_tile_slab_state(); init_copy_state(); } ArraySortedReadState::~ArraySortedReadState() { // Cancel copy thread copy_thread_canceled_ = true; for(int i=0; i<2; ++i) release_aio(i); // Wait for thread to be destroyed while(copy_thread_running_); // Join with the terminated thread pthread_join(copy_thread_, NULL); // Clean up free(subarray_); free(tile_coords_); free(tile_domain_); delete [] overflow_; for(int i=0; i<2; ++i) { delete [] aio_overflow_[i]; if(buffer_sizes_[i] != NULL) delete [] buffer_sizes_[i]; if(buffer_sizes_tmp_[i] != NULL) delete [] buffer_sizes_tmp_[i]; if(buffer_sizes_tmp_bak_[i] != NULL) delete [] buffer_sizes_tmp_bak_[i]; if(buffers_[i] != NULL) { for(int b=0; barray_schema()->coords_type(); if(type == TILEDB_INT32) { return read(); } else if(type == TILEDB_INT64) { return read(); } else if(type == TILEDB_FLOAT32) { return read(); } else if(type == TILEDB_FLOAT64) { return read(); } else { assert(0); return TILEDB_ASRS_ERR; } } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int ArraySortedReadState::init() { // Create buffers if(create_buffers() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Create AIO requests init_aio_requests(); // Initialize the mutexes and conditions if(pthread_mutex_init(&aio_mtx_, NULL)) { std::string errmsg = "Cannot initialize IO mutex"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } if(pthread_mutex_init(©_mtx_, NULL)) { std::string errmsg = "Cannot initialize copy mutex"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } if(pthread_mutex_init(&overflow_mtx_, NULL)) { std::string errmsg = "Cannot initialize overflow mutex"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } for(int i=0; i<2; ++i) { aio_cond_[i] = PTHREAD_COND_INITIALIZER; if(pthread_cond_init(&(aio_cond_[i]), NULL)) { std::string errmsg = "Cannot initialize IO mutex condition"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } copy_cond_[i] = PTHREAD_COND_INITIALIZER; if(pthread_cond_init(&(copy_cond_[i]), NULL)) { std::string errmsg = "Cannot initialize copy mutex condition"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } } overflow_cond_ = PTHREAD_COND_INITIALIZER; if(pthread_cond_init(&overflow_cond_, NULL)) { std::string errmsg = "Cannot initialize overflow mutex condition"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } // Initialize functors const ArraySchema* array_schema = array_->array_schema(); int mode = array_->mode(); int cell_order = array_schema->cell_order(); int tile_order = array_schema->tile_order(); int coords_type = array_schema->coords_type(); if(mode == TILEDB_ARRAY_READ_SORTED_ROW) { if(coords_type == TILEDB_INT32) { advance_cell_slab_ = advance_cell_slab_row_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; } else if(coords_type == TILEDB_INT64) { advance_cell_slab_ = advance_cell_slab_row_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; } else if(coords_type == TILEDB_FLOAT32) { advance_cell_slab_ = advance_cell_slab_row_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; } else if(coords_type == TILEDB_FLOAT64) { advance_cell_slab_ = advance_cell_slab_row_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; } else { assert(0); } } else { // mode == TILEDB_ARRAY_READ_SORTED_COL if(coords_type == TILEDB_INT32) { advance_cell_slab_ = advance_cell_slab_col_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; } else if(coords_type == TILEDB_INT64) { advance_cell_slab_ = advance_cell_slab_col_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; } else if(coords_type == TILEDB_FLOAT32) { advance_cell_slab_ = advance_cell_slab_col_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; } else if(coords_type == TILEDB_FLOAT64) { advance_cell_slab_ = advance_cell_slab_col_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; } else { assert(0); } } if(tile_order == TILEDB_ROW_MAJOR) { if(coords_type == TILEDB_INT32) calculate_tile_slab_info_ = calculate_tile_slab_info_row; else if(coords_type == TILEDB_INT64) calculate_tile_slab_info_ = calculate_tile_slab_info_row; else if(coords_type == TILEDB_FLOAT32) calculate_tile_slab_info_ = calculate_tile_slab_info_row; else if(coords_type == TILEDB_FLOAT64) calculate_tile_slab_info_ = calculate_tile_slab_info_row; else assert(0); } else { // tile_order == TILEDB_COL_MAJOR if(coords_type == TILEDB_INT32) calculate_tile_slab_info_ = calculate_tile_slab_info_col; else if(coords_type == TILEDB_INT64) calculate_tile_slab_info_ = calculate_tile_slab_info_col; else if(coords_type == TILEDB_FLOAT32) calculate_tile_slab_info_ = calculate_tile_slab_info_col; else if(coords_type == TILEDB_FLOAT64) calculate_tile_slab_info_ = calculate_tile_slab_info_col; else assert(0); } // Create the thread that will be handling all the copying if(pthread_create( ©_thread_, NULL, ArraySortedReadState::copy_handler, this)) { std::string errmsg = "Cannot create AIO thread"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } copy_thread_running_ = true; // Success return TILEDB_ASRS_OK; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ template void *ArraySortedReadState::advance_cell_slab_col_s(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int aid = ((ASRS_Data*) data)->id_; asrs->advance_cell_slab_col(aid); return NULL; } template void *ArraySortedReadState::advance_cell_slab_row_s(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int aid = ((ASRS_Data*) data)->id_; asrs->advance_cell_slab_row(aid); return NULL; } template void ArraySortedReadState::advance_cell_slab_col(int aid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; // Tile id int64_t cell_slab_num = tile_slab_info_[copy_id_].cell_slab_num_[tid]; T* current_coords = (T*) tile_slab_state_.current_coords_[aid]; const T* tile_slab = (const T*) tile_slab_norm_[copy_id_]; // Advance cell slab coordinates int d = 0; current_coords[d] += cell_slab_num; int64_t dim_overflow; for(int i=0; i tile_slab[2*(dim_num_-1)+1]) { tile_slab_state_.copy_tile_slab_done_[aid] = true; return; } // Calculate new tile and offset for the current coords update_current_tile_and_offset(aid); } template void ArraySortedReadState::advance_cell_slab_row(int aid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; // Tile id int64_t cell_slab_num = tile_slab_info_[copy_id_].cell_slab_num_[tid]; T* current_coords = (T*) tile_slab_state_.current_coords_[aid]; const T* tile_slab = (const T*) tile_slab_norm_[copy_id_]; // Advance cell slab coordinates int d = dim_num_-1; current_coords[d] += cell_slab_num; int64_t dim_overflow; for(int i=d; i>0; --i) { dim_overflow = (current_coords[i] - tile_slab[2*i]) / (tile_slab[2*i+1]-tile_slab[2*i]+1); current_coords[i-1] += dim_overflow; current_coords[i] -= dim_overflow * (tile_slab[2*i+1]-tile_slab[2*i]+1); } // Check if done if(current_coords[0] > tile_slab[1]) { tile_slab_state_.copy_tile_slab_done_[aid] = true; return; } // Calculate new tile and offset for the current coords update_current_tile_and_offset(aid); } void *ArraySortedReadState::aio_done(void* data) { // Retrieve data ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; // For easy reference int anum = (int) asrs->attribute_ids_.size(); const ArraySchema* array_schema = asrs->array_->array_schema(); // Check for overflow bool overflow = false; for(int i=0; ioverflow_still_[i] && asrs->aio_overflow_[id][i]) { overflow = true; break; } } // Handle overflow bool sparse = array_schema->dense(); if(overflow) { // OVERFLOW // Update buffer sizes for(int i=0, b=0; ivar_size(asrs->attribute_ids_[i])) { // FIXED if(asrs->aio_overflow_[id][i]) { // Expand buffer expand_buffer(asrs->buffers_[id][b], asrs->buffer_sizes_[id][b]); // Re-assign the buffer size for the fixed-sized offsets asrs->buffer_sizes_tmp_[id][b] = asrs->buffer_sizes_[id][b]; } else { // Backup sizes and zero them asrs->buffer_sizes_tmp_bak_[id][b] = asrs->buffer_sizes_tmp_[id][b]; asrs->buffer_sizes_tmp_[id][b] = 0; // Does not overflow any more asrs->overflow_still_[i] = false; } ++b; } else { // VAR if(asrs->aio_overflow_[id][i]) { // Expand offset buffer only in the case of sparse arrays if(sparse) expand_buffer(asrs->buffers_[id][b], asrs->buffer_sizes_[id][b]); // Re-assign the buffer size for the fixed-sized offsets asrs->buffer_sizes_tmp_[id][b] = asrs->buffer_sizes_[id][b]; ++b; // Expand variable-length cell buffers for both dense and sparse expand_buffer(asrs->buffers_[id][b], asrs->buffer_sizes_[id][b]); // Assign the new buffer size for the variable-sized values asrs->buffer_sizes_tmp_[id][b] = asrs->buffer_sizes_[id][b]; ++b; } else { // Backup sizes and zero them (fixed-sized offsets) asrs->buffer_sizes_tmp_bak_[id][b] = asrs->buffer_sizes_tmp_[id][b]; asrs->buffer_sizes_tmp_[id][b] = 0; ++b; // Backup sizes and zero them (variable-sized values) asrs->buffer_sizes_tmp_bak_[id][b] = asrs->buffer_sizes_tmp_[id][b]; asrs->buffer_sizes_tmp_[id][b] = 0; ++b; // Does not overflow any more asrs->overflow_still_[i] = false; } } } // Send the request again asrs->send_aio_request(id); } else { // NO OVERFLOW // Restore backup temporary buffer sizes for(int b=0; bbuffer_num_; ++b) { if(asrs->buffer_sizes_tmp_bak_[id][b] != 0) asrs->buffer_sizes_tmp_[id][b] = asrs->buffer_sizes_tmp_bak_[id][b]; } // Manage the mutexes and conditions asrs->release_aio(id); } return NULL; } bool ArraySortedReadState::aio_overflow(int aio_id) { // For easy reference int anum = (int) attribute_ids_.size(); for(int i=0; iattribute_ids(); coords_attr_i_ = -1; // For ease reference const ArraySchema* array_schema = array_->array_schema(); int attribute_num = array_schema->attribute_num(); // No need to do anything else in case the array is dense if(array_schema->dense()) return; // Find the coordinates index for(int i=0; i<(int)attribute_ids_.size(); ++i) { if(attribute_ids_[i] == attribute_num) { coords_attr_i_ = i; break; } } // If the coordinates index is not found, append coordinates attribute // to attribute ids. if(coords_attr_i_ == -1) { attribute_ids_.push_back(attribute_num); coords_attr_i_ = attribute_ids_.size() - 1; extra_coords_ = true; } else { // No extra coordinates appended extra_coords_ = false; } } void ArraySortedReadState::calculate_buffer_num() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); int attribute_num = array_schema->attribute_num(); // Calculate number of buffers buffer_num_ = 0; int attribute_id_num = (int) attribute_ids_.size(); for(int i=0; ivar_size(attribute_ids_[i])) { if(attribute_ids_[i] == attribute_num) coords_buf_i_ = i; // Buffer that holds the coordinates ++buffer_num_; } else { // Variable-sized attribute buffer_num_ += 2; } } } void ArraySortedReadState::calculate_buffer_sizes() { if(array_->array_schema()->dense()) calculate_buffer_sizes_dense(); else calculate_buffer_sizes_sparse(); } void ArraySortedReadState::calculate_buffer_sizes_dense() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); // Get cell number in a (full) tile slab int64_t tile_slab_cell_num; if(array_->mode() == TILEDB_ARRAY_READ_SORTED_ROW) tile_slab_cell_num = array_schema->tile_slab_row_cell_num(subarray_); else // TILEDB_ARRAY_READ_SORTED_COL tile_slab_cell_num = array_schema->tile_slab_col_cell_num(subarray_); // Calculate buffer sizes int attribute_id_num = (int) attribute_ids_.size(); for(int j=0; j<2; ++j) { buffer_sizes_[j] = new size_t[buffer_num_]; buffer_sizes_tmp_[j] = new size_t[buffer_num_]; buffer_sizes_tmp_bak_[j] = new size_t[buffer_num_]; for(int i=0, b=0; ivar_size(attribute_ids_[i])) { buffer_sizes_[j][b] = tile_slab_cell_num * array_schema->cell_size(attribute_ids_[i]); buffer_sizes_tmp_bak_[j][b] = 0; ++b; } else { // Variable-sized attribute buffer_sizes_[j][b] = tile_slab_cell_num * sizeof(size_t); buffer_sizes_tmp_bak_[j][b] = 0; ++b; buffer_sizes_[j][b] = 2 * tile_slab_cell_num * sizeof(size_t); buffer_sizes_tmp_bak_[j][b] = 0; ++b; } } } } void ArraySortedReadState::calculate_buffer_sizes_sparse() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); // Calculate buffer sizes int attribute_id_num = (int) attribute_ids_.size(); for(int j=0; j<2; ++j) { buffer_sizes_[j] = new size_t[buffer_num_]; buffer_sizes_tmp_[j] = new size_t[buffer_num_]; buffer_sizes_tmp_bak_[j] = new size_t[buffer_num_]; for(int i=0, b=0; ivar_size(attribute_ids_[i])) { // Variable-sized buffer buffer_sizes_[j][b] = 2*TILEDB_ASRS_INIT_BUFFER_SIZE; buffer_sizes_tmp_bak_[j][b] = 0; ++b; } } } } template void *ArraySortedReadState::calculate_cell_slab_info_col_col_s(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; int tid = ((ASRS_Data*) data)->id_2_; asrs->calculate_cell_slab_info_col_col(id, tid); return NULL; } template void *ArraySortedReadState::calculate_cell_slab_info_col_row_s(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; int tid = ((ASRS_Data*) data)->id_2_; asrs->calculate_cell_slab_info_col_row(id, tid); return NULL; } template void *ArraySortedReadState::calculate_cell_slab_info_row_col_s(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; int tid = ((ASRS_Data*) data)->id_2_; asrs->calculate_cell_slab_info_row_col(id, tid); return NULL; } template void *ArraySortedReadState::calculate_cell_slab_info_row_row_s(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; int tid = ((ASRS_Data*) data)->id_2_; asrs->calculate_cell_slab_info_row_row(id, tid); return NULL; } template void ArraySortedReadState::calculate_cell_slab_info_col_col( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* range_overlap = (const T*) tile_slab_info_[id].range_overlap_[tid]; const T* tile_domain = (const T*) tile_domain_; int64_t tile_num, cell_num; // Calculate number of cells in cell slab cell_num = range_overlap[1] - range_overlap[0] + 1; for(int i=0; i void ArraySortedReadState::calculate_cell_slab_info_row_row( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* range_overlap = (const T*) tile_slab_info_[id].range_overlap_[tid]; const T* tile_domain = (const T*) tile_domain_; int64_t tile_num, cell_num; // Calculate number of cells in cell slab cell_num = range_overlap[2*(dim_num_-1)+1] - range_overlap[2*(dim_num_-1)] +1; for(int i=dim_num_-1; i>0; --i) { tile_num = tile_domain[2*i+1] - tile_domain[2*i] + 1; if(tile_num == 1) cell_num *= range_overlap[2*(i-1)+1] - range_overlap[2*(i-1)] + 1; else break; } tile_slab_info_[id].cell_slab_num_[tid] = cell_num; // Calculate size of a cell slab per attribute for(int aid=0; aid=0; --i) { cell_offset *= (range_overlap[2*(i+1)+1] - range_overlap[2*(i+1)] + 1); tile_slab_info_[id].cell_offset_per_dim_[tid][i] = cell_offset; } } template void ArraySortedReadState::calculate_cell_slab_info_col_row( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* range_overlap = (const T*) tile_slab_info_[id].range_overlap_[tid]; // Calculate number of cells in cell slab tile_slab_info_[id].cell_slab_num_[tid] = 1; // Calculate size of a cell slab per attribute for(int aid=0; aid=0; --i) { cell_offset *= (range_overlap[2*(i+1)+1] - range_overlap[2*(i+1)] + 1); tile_slab_info_[id].cell_offset_per_dim_[tid][i] = cell_offset; } } template void ArraySortedReadState::calculate_cell_slab_info_row_col( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* range_overlap = (const T*) tile_slab_info_[id].range_overlap_[tid]; // Calculate number of cells in cell slab tile_slab_info_[id].cell_slab_num_[tid] = 1; // Calculate size of a cell slab per attribute for(int aid=0; aid void ArraySortedReadState::calculate_tile_domain(int id) { // Initializations tile_coords_ = malloc(coords_size_); tile_domain_ = malloc(2*coords_size_); // For easy reference const T* tile_slab = (const T*) tile_slab_norm_[id]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); T* tile_coords = (T*) tile_coords_; T* tile_domain = (T*) tile_domain_; // Calculate tile domain and initial tile coordinates for(int i=0; i void ArraySortedReadState::calculate_tile_slab_info(int id) { // Calculate number of tiles, if they are not already calculated if(tile_slab_info_[id].tile_num_ == -1) init_tile_slab_info(id); // Calculate tile domain, if not calculated yet if(tile_domain_ == NULL) calculate_tile_domain(id); // Reset tile coordinates reset_tile_coords(); // Calculate tile slab info ASRS_Data asrs_data = { id, 0, this }; (*calculate_tile_slab_info_)(&asrs_data); } template void *ArraySortedReadState::calculate_tile_slab_info_col(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; asrs->calculate_tile_slab_info_col(id); return NULL; } template void ArraySortedReadState::calculate_tile_slab_info_col(int id) { // For easy reference const T* tile_domain = (const T*) tile_domain_; T* tile_coords = (T*) tile_coords_; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); T** range_overlap = (T**) tile_slab_info_[id].range_overlap_; const T* tile_slab = (const T*) tile_slab_norm_[id]; int64_t tile_offset, tile_cell_num, total_cell_num = 0; int anum = (int) attribute_ids_.size(); int d; // Iterate over all tiles in the tile domain int64_t tid=0; // Tile id while(tile_coords[dim_num_-1] <= tile_domain[2*(dim_num_-1)+1]) { // Calculate range overlap, number of cells in the tile tile_cell_num = 1; for(int i=0; i tile_domain[2*d+1]) { tile_coords[d] = tile_domain[2*d]; ++tile_coords[++d]; } // Advance tile id ++tid; } } template void *ArraySortedReadState::calculate_tile_slab_info_row(void* data) { ArraySortedReadState* asrs = ((ASRS_Data*) data)->asrs_; int id = ((ASRS_Data*) data)->id_; asrs->calculate_tile_slab_info_row(id); return NULL; } template void ArraySortedReadState::calculate_tile_slab_info_row(int id) { // For easy reference const T* tile_domain = (const T*) tile_domain_; T* tile_coords = (T*) tile_coords_; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); T** range_overlap = (T**) tile_slab_info_[id].range_overlap_; const T* tile_slab = (const T*) tile_slab_norm_[id]; int64_t tile_offset, tile_cell_num, total_cell_num = 0; int anum = (int) attribute_ids_.size(); int d; // Iterate over all tiles in the tile domain int64_t tid=0; // Tile id while(tile_coords[0] <= tile_domain[1]) { // Calculate range overlap, number of cells in the tile tile_cell_num = 1; for(int i=0; i=0; --i) { tile_offset *= (tile_domain[2*(i+1)+1] - tile_domain[2*(i+1)] + 1); tile_slab_info_[id].tile_offset_per_dim_[i] = tile_offset; } // Calculate cell slab info ASRS_Data asrs_data = { id, tid, this }; (*calculate_cell_slab_info_)(&asrs_data); // Calculate start offsets for(int aid=0; aid 0 && tile_coords[d] > tile_domain[2*d+1]) { tile_coords[d] = tile_domain[2*d]; ++tile_coords[--d]; } // Advance tile id ++tid; } } void *ArraySortedReadState::copy_handler(void* context) { // For easy reference ArraySortedReadState* asrs = (ArraySortedReadState*) context; // This will enter an indefinite loop that will handle all incoming copy // requests int coords_type = asrs->array_->array_schema()->coords_type(); if(asrs->array_->array_schema()->dense()) { // DENSE if(coords_type == TILEDB_INT32) asrs->handle_copy_requests_dense(); else if(coords_type == TILEDB_INT64) asrs->handle_copy_requests_dense(); else if(coords_type == TILEDB_FLOAT32) asrs->handle_copy_requests_dense(); else if(coords_type == TILEDB_FLOAT64) asrs->handle_copy_requests_dense(); else assert(0); } else { // SPARSE if(coords_type == TILEDB_INT32) asrs->handle_copy_requests_sparse(); else if(coords_type == TILEDB_INT64) asrs->handle_copy_requests_sparse(); else if(coords_type == TILEDB_FLOAT32) asrs->handle_copy_requests_sparse(); else if(coords_type == TILEDB_FLOAT64) asrs->handle_copy_requests_sparse(); else assert(0); } // Return return NULL; } void ArraySortedReadState::copy_tile_slab_dense() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); // Copy tile slab for each attribute separately for(int i=0, b=0; i<(int)attribute_ids_.size(); ++i) { if(!array_schema->var_size(attribute_ids_[i])) { copy_tile_slab_dense(i, b); ++b; } else { copy_tile_slab_dense_var(i, b); b += 2; } } } void ArraySortedReadState::copy_tile_slab_dense(int aid, int bid) { // Exit if copy is done for this attribute if(tile_slab_state_.copy_tile_slab_done_[aid]) { copy_state_.buffer_sizes_[bid] = 0; // Nothing written return; } // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; size_t& buffer_offset = copy_state_.buffer_offsets_[bid]; size_t buffer_size = copy_state_.buffer_sizes_[bid]; char* buffer = (char*) copy_state_.buffers_[bid]; char* local_buffer = (char*) buffers_[copy_id_][bid]; ASRS_Data asrs_data = { aid, 0, this }; // Iterate over the tile slab cells for(;;) { // For easy reference size_t cell_slab_size = tile_slab_info_[copy_id_].cell_slab_size_[aid][tid]; size_t& local_buffer_offset = tile_slab_state_.current_offsets_[aid]; // Handle overflow if(buffer_offset + cell_slab_size > buffer_size) { overflow_[aid] = true; break; } // Copy cell slab memcpy( buffer + buffer_offset, local_buffer + local_buffer_offset, cell_slab_size); // Update buffer offset buffer_offset += cell_slab_size; // Prepare for new cell slab (*advance_cell_slab_)(&asrs_data); // Terminating condition if(tile_slab_state_.copy_tile_slab_done_[aid]) break; } // Set user buffer size buffer_size = buffer_offset; } void ArraySortedReadState::copy_tile_slab_dense_var(int aid, int bid) { // Exit if copy is done for this attribute if(tile_slab_state_.copy_tile_slab_done_[aid]) { copy_state_.buffer_sizes_[bid] = 0; // Nothing written copy_state_.buffer_sizes_[bid+1] = 0; // Nothing written return; } // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; size_t cell_slab_size_var; size_t& buffer_offset = copy_state_.buffer_offsets_[bid]; size_t& buffer_offset_var = copy_state_.buffer_offsets_[bid+1]; size_t buffer_size = copy_state_.buffer_sizes_[bid]; size_t buffer_size_var = copy_state_.buffer_sizes_[bid+1]; char* buffer = (char*) copy_state_.buffers_[bid]; char* buffer_var = (char*) copy_state_.buffers_[bid+1]; char* local_buffer_var = (char*) buffers_[copy_id_][bid+1]; size_t local_buffer_size = buffer_sizes_tmp_[copy_id_][bid]; size_t local_buffer_var_size = buffer_sizes_tmp_[copy_id_][bid+1]; size_t* local_buffer_s = (size_t*) buffers_[copy_id_][bid]; int64_t cell_num_in_buffer = local_buffer_size / sizeof(size_t); size_t var_offset = buffer_offset_var; ASRS_Data asrs_data = { aid, 0, this }; // For all overlapping tiles, in a round-robin fashion for(;;) { // For easy reference size_t cell_slab_size = tile_slab_info_[copy_id_].cell_slab_size_[aid][tid]; int64_t cell_num_in_slab = cell_slab_size / sizeof(size_t); size_t& local_buffer_offset = tile_slab_state_.current_offsets_[aid]; // Handle overflow if(buffer_offset + cell_slab_size > buffer_size) { overflow_[aid] = true; break; } // Calculate variable cell slab size int64_t cell_start = local_buffer_offset / sizeof(size_t); int64_t cell_end = cell_start + cell_num_in_slab; cell_slab_size_var = (cell_end == cell_num_in_buffer) ? local_buffer_var_size - local_buffer_s[cell_start] : local_buffer_s[cell_end] - local_buffer_s[cell_start]; // Handle overflow for the the variable-length buffer if(buffer_offset_var + cell_slab_size_var > buffer_size_var) { overflow_[aid] = true; break; } // Copy fixed-sized offsets for(int64_t i=cell_start; iarray_schema(); // Copy tile slab for each attribute separately for(int i=0, b=0; i<(int)attribute_ids_.size(); ++i) { if(!array_schema->var_size(attribute_ids_[i])) { // FIXED // Make sure not to copy coordinates if the user has not requested them if(i != coords_attr_i_ || !extra_coords_) copy_tile_slab_sparse(i, b); ++b; } else { // VAR copy_tile_slab_sparse_var(i, b); b += 2; } } } void ArraySortedReadState::copy_tile_slab_sparse(int aid, int bid) { // Exit if copy is done for this attribute if(tile_slab_state_.copy_tile_slab_done_[aid]) { copy_state_.buffer_sizes_[bid] = 0; // Nothing written return; } // For easy reference size_t cell_size = array_->array_schema()->cell_size(attribute_ids_[aid]); size_t& buffer_offset = copy_state_.buffer_offsets_[bid]; size_t buffer_size = copy_state_.buffer_sizes_[bid]; char* buffer = (char*) copy_state_.buffers_[bid]; char* local_buffer = (char*) buffers_[copy_id_][bid]; size_t local_buffer_offset; int64_t cell_num = buffer_sizes_tmp_[copy_id_][coords_buf_i_] / coords_size_; int64_t& current_cell_pos = tile_slab_state_.current_cell_pos_[aid]; // Iterate over the remaining tile slab cells in a sorted order for(; current_cell_pos buffer_size) { overflow_[aid] = true; break; } // Calculate new local buffer offset local_buffer_offset = cell_pos_[current_cell_pos] * cell_size; // Copy cell slab memcpy( buffer + buffer_offset, local_buffer + local_buffer_offset, cell_size); // Update buffer offset buffer_offset += cell_size; } // Mark tile slab as done if(current_cell_pos == cell_num) tile_slab_state_.copy_tile_slab_done_[aid] = true; // Set user buffer size buffer_size = buffer_offset; } void ArraySortedReadState::copy_tile_slab_sparse_var(int aid, int bid) { // Exit if copy is done for this attribute if(tile_slab_state_.copy_tile_slab_done_[aid]) { copy_state_.buffer_sizes_[bid] = 0; // Nothing written copy_state_.buffer_sizes_[bid+1] = 0; // Nothing written return; } // For easy reference size_t cell_size = sizeof(size_t); size_t cell_size_var; size_t& buffer_offset = copy_state_.buffer_offsets_[bid]; size_t& buffer_offset_var = copy_state_.buffer_offsets_[bid+1]; size_t buffer_size = copy_state_.buffer_sizes_[bid]; size_t buffer_size_var = copy_state_.buffer_sizes_[bid+1]; char* buffer = (char*) copy_state_.buffers_[bid]; char* buffer_var = (char*) copy_state_.buffers_[bid+1]; char* local_buffer_var = (char*) buffers_[copy_id_][bid+1]; size_t local_buffer_var_size = buffer_sizes_tmp_[copy_id_][bid+1]; size_t* local_buffer_s = (size_t*) buffers_[copy_id_][bid]; int64_t cell_num = buffer_sizes_tmp_[copy_id_][coords_buf_i_] / coords_size_; int64_t& current_cell_pos = tile_slab_state_.current_cell_pos_[aid]; // Iterate over the remaining tile slab cells in a sorted order for(; current_cell_pos buffer_size) { overflow_[aid] = true; break; } // Calculate variable cell size int64_t cell_start = cell_pos_[current_cell_pos]; int64_t cell_end = cell_start + 1; cell_size_var = (cell_end == cell_num) ? local_buffer_var_size - local_buffer_s[cell_start] : local_buffer_s[cell_end] - local_buffer_s[cell_start]; // Handle overflow for the the variable-length buffer if(buffer_offset_var + cell_size_var > buffer_size_var) { overflow_[aid] = true; break; } // Copy fixed-sized offset memcpy( buffer + buffer_offset, &buffer_offset_var, sizeof(size_t)); buffer_offset += sizeof(size_t); // Copy variable-sized values memcpy( buffer_var + buffer_offset_var, local_buffer_var + local_buffer_s[cell_start], cell_size_var); buffer_offset_var += cell_size_var; } // Mark tile slab as done if(current_cell_pos == cell_num) tile_slab_state_.copy_tile_slab_done_[aid] = true; // Set user buffer sizes buffer_size = buffer_offset; buffer_size_var = buffer_offset_var; } int ArraySortedReadState::create_buffers() { for(int j=0; j<2; ++j) { buffers_[j] = (void**) malloc(buffer_num_ * sizeof(void*)); if(buffers_[j] == NULL) { std::string errmsg = "Cannot create local buffers"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } for(int b=0; b < buffer_num_; ++b) { buffers_[j][b] = malloc(buffer_sizes_[j][b]); if(buffers_[j][b] == NULL) { std::string errmsg = "Cannot allocate local buffer"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } } } // Success return TILEDB_ASRS_OK; } void ArraySortedReadState::free_copy_state() { if(copy_state_.buffer_offsets_ != NULL) delete [] copy_state_.buffer_offsets_; } void ArraySortedReadState::free_tile_slab_info() { // Do nothing in the case of sparse arrays if(!array_->array_schema()->dense()) return; // For easy reference int anum = (int) attribute_ids_.size(); // Free for(int i=0; i<2; ++i) { int64_t tile_num = tile_slab_info_[i].tile_num_; if(tile_slab_info_[i].cell_offset_per_dim_ != NULL) { for(int j=0; j int64_t ArraySortedReadState::get_cell_id(int aid) { // For easy reference const T* current_coords = (const T*) tile_slab_state_.current_coords_[aid]; int64_t tid = tile_slab_state_.current_tile_[aid]; const T* range_overlap = (const T*) tile_slab_info_[copy_id_].range_overlap_[tid]; int64_t* cell_offset_per_dim = tile_slab_info_[copy_id_].cell_offset_per_dim_[tid]; // Calculate cell id int64_t cid = 0; for(int i=0; i int64_t ArraySortedReadState::get_tile_id(int aid) { // For easy reference const T* current_coords = (const T*) tile_slab_state_.current_coords_[aid]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); int64_t* tile_offset_per_dim = tile_slab_info_[copy_id_].tile_offset_per_dim_; // Calculate tile id int64_t tid = 0; for(int i=0; i void ArraySortedReadState::handle_copy_requests_dense() { // Handle copy requests indefinitely for(;;) { // Wait for AIO wait_aio(copy_id_); // Kill thread, after releasing any blocked resources if(copy_thread_canceled_) { copy_thread_running_ = false; return; } // Reset the tile slab state if(copy_tile_slab_done()) reset_tile_slab_state(); // Start the copy copy_tile_slab_dense(); // Wait in case of overflow if(overflow()) { block_overflow(); block_aio(copy_id_); release_copy(0); release_copy(1); wait_overflow(); continue; } // Copy is done block_aio(copy_id_); release_copy(copy_id_); copy_id_ = (copy_id_ + 1) % 2; } } template void ArraySortedReadState::handle_copy_requests_sparse() { // Handle copy requests indefinitely for(;;) { // Wait for AIO wait_aio(copy_id_); // Kill thread, after releasing any blocked resources if(copy_thread_canceled_) { copy_thread_running_ = false; return; } // Sort the cell positions if(copy_tile_slab_done()) { reset_tile_slab_state(); sort_cell_pos(); } // Start the copy copy_tile_slab_sparse(); // Wait in case of overflow if(overflow()) { block_overflow(); block_aio(copy_id_); release_copy(0); release_copy(1); wait_overflow(); continue; } // Copy is done block_aio(copy_id_); release_copy(copy_id_); copy_id_ = (copy_id_ + 1) % 2; } } void ArraySortedReadState::init_aio_requests() { for(int i=0; i<2; ++i) { aio_data_[i] = { i, 0, this }; memset(&aio_request_[i], 0, sizeof(AIO_Request));; aio_request_[i].buffer_sizes_ = buffer_sizes_tmp_[i]; aio_request_[i].buffers_ = buffers_[i]; aio_request_[i].mode_ = TILEDB_ARRAY_READ; aio_request_[i].subarray_ = tile_slab_[i]; aio_request_[i].completion_handle_ = aio_done; aio_request_[i].completion_data_ = &(aio_data_[i]); aio_request_[i].overflow_ = aio_overflow_[i]; aio_request_[i].status_ = &(aio_status_[i]); } } void ArraySortedReadState::init_copy_state() { copy_state_.buffer_sizes_ = NULL; copy_state_.buffers_ = NULL; copy_state_.buffer_offsets_ = new size_t[buffer_num_]; for(int i=0; iarray_schema()->dense()) return; // For easy reference int anum = (int) attribute_ids_.size(); // Initialize for(int i=0; i<2; ++i) { tile_slab_info_[i].cell_offset_per_dim_ = NULL; tile_slab_info_[i].cell_slab_size_ = new size_t*[anum]; tile_slab_info_[i].cell_slab_num_ = NULL; tile_slab_info_[i].range_overlap_ = NULL; tile_slab_info_[i].start_offsets_ = new size_t*[anum]; tile_slab_info_[i].tile_offset_per_dim_ = new int64_t[dim_num_]; for(int j=0; j void ArraySortedReadState::init_tile_slab_info(int id) { // Sanity check assert(array_->array_schema()->dense()); // For easy reference int anum = (int) attribute_ids_.size(); // Calculate tile number int64_t tile_num = array_->array_schema()->tile_num(tile_slab_[id]); // Initializations tile_slab_info_[id].cell_offset_per_dim_ = new int64_t*[tile_num]; tile_slab_info_[id].cell_slab_num_ = new int64_t[tile_num]; tile_slab_info_[id].range_overlap_ = new void*[tile_num]; for(int64_t i=0; iarray_schema()->dense(); // Both for dense and sparse tile_slab_state_.copy_tile_slab_done_ = new bool[anum]; for(int i=0; i bool ArraySortedReadState::next_tile_slab_dense_col() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); const T* domain = static_cast(array_schema->domain()); const T* tile_extents = static_cast(array_schema->tile_extents()); T* tile_slab[2]; T* tile_slab_norm = static_cast(tile_slab_norm_[aio_id_]); for(int i=0; i<2; ++i) tile_slab[i] = static_cast(tile_slab_[i]); int prev_id = (aio_id_+1)%2; T tile_start; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][2*(dim_num_-1) + 1] == subarray[2*(dim_num_-1) + 1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][2*(dim_num_-1)] = subarray[2*(dim_num_-1)]; T upper = subarray[2*(dim_num_-1)] + tile_extents[dim_num_-1]; T cropped_upper = (upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1] * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = std::min(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i(aio_id_); // Mark this tile slab as initialized tile_slab_init_[aio_id_] = true; // Success return true; } template bool ArraySortedReadState::next_tile_slab_dense_row() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); const T* domain = static_cast(array_schema->domain()); const T* tile_extents = static_cast(array_schema->tile_extents()); T* tile_slab[2]; T* tile_slab_norm = static_cast(tile_slab_norm_[aio_id_]); for(int i=0; i<2; ++i) tile_slab[i] = static_cast(tile_slab_[i]); int prev_id = (aio_id_+1)%2; T tile_start; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][1] == subarray[1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][0] = subarray[0]; T upper = subarray[0] + tile_extents[0]; T cropped_upper = (upper - domain[0]) / tile_extents[0] * tile_extents[0] + domain[0]; tile_slab[aio_id_][1] = std::min(cropped_upper - 1, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i(aio_id_); // Mark this tile slab as initialized tile_slab_init_[aio_id_] = true; // Success return true; } template bool ArraySortedReadState::next_tile_slab_sparse_col() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); const T* domain = static_cast(array_schema->domain()); const T* tile_extents = static_cast(array_schema->tile_extents()); T* tile_slab[2]; for(int i=0; i<2; ++i) tile_slab[i] = static_cast(tile_slab_[i]); int prev_id = (aio_id_+1)%2; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][2*(dim_num_-1) + 1] == subarray[2*(dim_num_-1) + 1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][2*(dim_num_-1)] = subarray[2*(dim_num_-1)]; T upper = subarray[2*(dim_num_-1)] + tile_extents[dim_num_-1]; T cropped_upper = (upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1] * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = std::min(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i bool ArraySortedReadState::next_tile_slab_sparse_col() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const float* subarray = (const float*) subarray_; const float* domain = (const float*) array_schema->domain(); const float* tile_extents = (const float*) array_schema->tile_extents(); float* tile_slab[2]; for(int i=0; i<2; ++i) tile_slab[i] = (float*) tile_slab_[i]; int prev_id = (aio_id_+1)%2; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][2*(dim_num_-1) + 1] == subarray[2*(dim_num_-1) + 1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][2*(dim_num_-1)] = subarray[2*(dim_num_-1)]; float upper = subarray[2*(dim_num_-1)] + tile_extents[dim_num_-1]; float cropped_upper = floor((upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1]) * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = std::min(cropped_upper - FLT_MIN, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i bool ArraySortedReadState::next_tile_slab_sparse_col() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const double* subarray = (const double*) subarray_; const double* domain = (const double*) array_schema->domain(); const double* tile_extents = (const double*) array_schema->tile_extents(); double* tile_slab[2]; for(int i=0; i<2; ++i) tile_slab[i] = (double*) tile_slab_[i]; int prev_id = (aio_id_+1)%2; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][2*(dim_num_-1) + 1] == subarray[2*(dim_num_-1) + 1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][2*(dim_num_-1)] = subarray[2*(dim_num_-1)]; double upper = subarray[2*(dim_num_-1)] + tile_extents[dim_num_-1]; double cropped_upper = floor((upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1]) * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = std::min(cropped_upper - DBL_MIN, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i bool ArraySortedReadState::next_tile_slab_sparse_row() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); const T* domain = static_cast(array_schema->domain()); const T* tile_extents = static_cast(array_schema->tile_extents()); T* tile_slab[2]; for(int i=0; i<2; ++i) tile_slab[i] = static_cast(tile_slab_[i]); int prev_id = (aio_id_+1)%2; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][1] == subarray[1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][0] = subarray[0]; T upper = subarray[0] + tile_extents[0]; T cropped_upper = (upper - domain[0]) / tile_extents[0] * tile_extents[0] + domain[0]; tile_slab[aio_id_][1] = std::min(cropped_upper - 1, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i bool ArraySortedReadState::next_tile_slab_sparse_row() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const float* subarray = (const float*) subarray_; const float* domain = (const float*) array_schema->domain(); const float* tile_extents = (const float*) array_schema->tile_extents(); float* tile_slab[2]; for(int i=0; i<2; ++i) tile_slab[i] = (float*) tile_slab_[i]; int prev_id = (aio_id_+1)%2; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][1] == subarray[1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][0] = subarray[0]; float upper = subarray[0] + tile_extents[0]; float cropped_upper = floor((upper - domain[0]) / tile_extents[0]) * tile_extents[0] + domain[0]; tile_slab[aio_id_][1] = std::min(cropped_upper - FLT_MIN, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i bool ArraySortedReadState::next_tile_slab_sparse_row() { // Quick check if done if(read_tile_slabs_done_) return false; // If the AIO needs to be resumed, exit (no need for a new tile slab) if(resume_aio_) { resume_aio_ = false; return true; } // Wait for the previous copy on aio_id_ buffer to be consumed wait_copy(aio_id_); // Block copy block_copy(aio_id_); // For easy reference const ArraySchema* array_schema = array_->array_schema(); const double* subarray = (const double*) subarray_; const double* domain = (const double*) array_schema->domain(); const double* tile_extents = (const double*) array_schema->tile_extents(); double* tile_slab[2]; for(int i=0; i<2; ++i) tile_slab[i] = (double*) tile_slab_[i]; int prev_id = (aio_id_+1)%2; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][1] == subarray[1]) { read_tile_slabs_done_ = true; return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[aio_id_][0] = subarray[0]; double upper = subarray[0] + tile_extents[0]; double cropped_upper = floor((upper - domain[0]) / tile_extents[0]) * tile_extents[0] + domain[0]; tile_slab[aio_id_][1] = std::min(cropped_upper - DBL_MIN, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i int ArraySortedReadState::read() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); int mode = array_->mode(); if(mode == TILEDB_ARRAY_READ_SORTED_COL) { if(array_schema->dense()) return read_dense_sorted_col(); else return read_sparse_sorted_col(); } else if(mode == TILEDB_ARRAY_READ_SORTED_ROW) { if(array_schema->dense()) return read_dense_sorted_row(); else return read_sparse_sorted_row(); } else { assert(0); // The code should never reach here return TILEDB_ASRS_ERR; } } template int ArraySortedReadState::read_dense_sorted_col() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); // Check if this can be satisfied with a default read if(array_schema->cell_order() == TILEDB_COL_MAJOR && array_schema->is_contained_in_tile_slab_row(subarray)) return array_->read_default( copy_state_.buffers_, copy_state_.buffer_sizes_); // Iterate over each tile slab while(next_tile_slab_dense_col()) { // Read the next tile slab with the default cell order if(read_tile_slab() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Handle overflow if(resume_aio_) break; } // Wait for copy to finish int copy_id = (resume_aio_) ? aio_id_ : (aio_id_ + 1) % 2; wait_copy(copy_id); // Assign the true buffer sizes for(int i=0; i int ArraySortedReadState::read_dense_sorted_row() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); // Check if this can be satisfied with a default read if(array_schema->cell_order() == TILEDB_ROW_MAJOR && array_schema->is_contained_in_tile_slab_col(subarray)) return array_->read_default( copy_state_.buffers_, copy_state_.buffer_sizes_); // Iterate over each tile slab while(next_tile_slab_dense_row()) { // Read the next tile slab with the default cell order if(read_tile_slab() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Handle overflow if(resume_aio_) break; } // Wait for copy and AIO to finish int copy_id = (resume_aio_) ? aio_id_ : (aio_id_ + 1) % 2; wait_copy(copy_id); // Assign the true buffer sizes for(int i=0; i int ArraySortedReadState::read_sparse_sorted_col() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); // Check if this can be satisfied with a default read if(array_schema->cell_order() == TILEDB_COL_MAJOR && array_schema->is_contained_in_tile_slab_row(subarray)) return array_->read_default( copy_state_.buffers_, copy_state_.buffer_sizes_); // Iterate over each tile slab while(next_tile_slab_sparse_col()) { // Read the next tile slab with the default cell order if(read_tile_slab() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Handle overflow if(resume_aio_) break; } // Wait for copy to finish int copy_id = (resume_aio_) ? aio_id_ : (aio_id_ + 1) % 2; wait_copy(copy_id); // Assign the true buffer sizes int buffer_num = buffer_num_ - (int) extra_coords_; for(int i=0; i int ArraySortedReadState::read_sparse_sorted_row() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); // Check if this can be satisfied with a default read if(array_schema->cell_order() == TILEDB_ROW_MAJOR && array_schema->is_contained_in_tile_slab_col(subarray)) return array_->read_default( copy_state_.buffers_, copy_state_.buffer_sizes_); // Iterate over each tile slab while(next_tile_slab_sparse_row()) { // Read the next tile slab with the default cell order if(read_tile_slab() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Handle overflow if(resume_aio_) break; } // Wait for copy and AIO to finish int copy_id = (resume_aio_) ? aio_id_ : (aio_id_ + 1) % 2; wait_copy(copy_id); // Assign the true buffer sizes int buffer_num = buffer_num_ - (int) extra_coords_; for(int i=0; i void ArraySortedReadState::reset_tile_coords() { T* tile_coords = (T*) tile_coords_; for(int i=0; i void ArraySortedReadState::reset_tile_slab_state() { // For easy reference int anum = (int) attribute_ids_.size(); bool dense = array_->array_schema()->dense(); // Both dense and sparse for(int i=0; iarray_clone(); // Sanity check assert(array_clone != NULL); // Send the AIO request to the clone array if(array_clone->aio_read(&(aio_request_[aio_id])) != TILEDB_AR_OK) { tiledb_asrs_errmsg = tiledb_ar_errmsg; return TILEDB_ASRS_ERR; } // Success return TILEDB_ASRS_OK; } template void ArraySortedReadState::sort_cell_pos() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); int dim_num = array_schema->dim_num(); int64_t cell_num = buffer_sizes_tmp_[copy_id_][coords_buf_i_] / coords_size_; int mode = array_->mode(); const T* buffer = static_cast(buffers_[copy_id_][coords_buf_i_]); // Populate cell_pos cell_pos_.resize(cell_num); for(int i=0; i(buffer, dim_num)); } else { // mode == TILEDB_ARRAY_READ_SORTED_COL // Sort cell positions SORT( cell_pos_.begin(), cell_pos_.end(), SmallerCol(buffer, dim_num)); } } int ArraySortedReadState::unlock_aio_mtx() { if(pthread_mutex_unlock(&aio_mtx_)) { std::string errmsg = "Cannot unlock AIO mutex"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } // Success return TILEDB_ASRS_OK; } int ArraySortedReadState::unlock_copy_mtx() { if(pthread_mutex_unlock(©_mtx_)) { std::string errmsg = "Cannot unlock copy mutex"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } // Success return TILEDB_ASRS_OK; } int ArraySortedReadState::unlock_overflow_mtx() { if(pthread_mutex_unlock(&overflow_mtx_)) { std::string errmsg = "Cannot unlock overflow mutex"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } // Success return TILEDB_ASRS_OK; } template void ArraySortedReadState::update_current_tile_and_offset(int aid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; size_t& current_offset = tile_slab_state_.current_offsets_[aid]; int64_t cid; // Calculate the new tile id tid = get_tile_id(aid); // Calculate the cell id cid = get_cell_id(aid); // Calculate new offset current_offset = tile_slab_info_[copy_id_].start_offsets_[aid][tid] + cid * attribute_sizes_[aid]; } int ArraySortedReadState::wait_aio(int id) { // Lock AIO mutex if(lock_aio_mtx() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Wait to be signaled while(wait_aio_[id]) { if(pthread_cond_wait(&(aio_cond_[id]), &aio_mtx_)) { std::string errmsg = "Cannot wait on IO mutex condition"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } } // Unlock AIO mutex if(unlock_aio_mtx() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Success return TILEDB_ASRS_OK; } int ArraySortedReadState::wait_copy(int id) { // Lock copy mutex if(lock_copy_mtx() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Wait to be signaled while(wait_copy_[id]) { if(pthread_cond_wait(&(copy_cond_[id]), ©_mtx_)) { std::string errmsg = "Cannot wait on copy mutex condition"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } } // Unlock copy mutex if(unlock_copy_mtx() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Success return TILEDB_ASRS_OK; } int ArraySortedReadState::wait_overflow() { // Lock overflow mutex if(lock_overflow_mtx() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Wait to be signaled while(overflow()) { if(pthread_cond_wait(&overflow_cond_, &overflow_mtx_)) { std::string errmsg = "Cannot wait on IO mutex condition"; PRINT_ERROR(errmsg); tiledb_asrs_errmsg = TILEDB_ASRS_ERRMSG + errmsg; return TILEDB_ASRS_ERR; } } // Unlock overflow mutex if(unlock_overflow_mtx() != TILEDB_ASRS_OK) return TILEDB_ASRS_ERR; // Success return TILEDB_ASRS_OK; } // Explicit template instantiations template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_row(); template int ArraySortedReadState::read_dense_sorted_row(); template int ArraySortedReadState::read_dense_sorted_row(); template int ArraySortedReadState::read_dense_sorted_row(); genomicsdb-0.0~git20231212.9d7ddd0/core/src/array/array_sorted_write_state.cc000066400000000000000000001745001453617025200266300ustar00rootroot00000000000000/** * @file array_sorted_write_state.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the ArraySortedWriteState class. */ #include "array_sorted_write_state.h" #include "math.h" #include "utils.h" #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_ASWS_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_asws_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ArraySortedWriteState::ArraySortedWriteState( Array* array) : array_(array), attribute_ids_(array->attribute_ids()) { // For easy reference const ArraySchema* array_schema = array_->array_schema(); int anum = (int) attribute_ids_.size(); // Initializations aio_cnt_ = 0; aio_id_ = 0; aio_thread_running_ = false; aio_thread_canceled_ = false; coords_size_ = array_schema->coords_size(); copy_id_ = 0; dim_num_ = array_schema->dim_num(); tile_coords_ = NULL; tile_domain_ = NULL; buffer_sizes_ = NULL; buffers_ = NULL; for(int i=0; i<2; ++i) { tile_slab_[i] = malloc(2*coords_size_); tile_slab_norm_[i] = malloc(2*coords_size_); tile_slab_init_[i] = false; wait_copy_[i] = true; wait_aio_[i] = false; } for(int i=0; ivar_size(attribute_ids_[i])) attribute_sizes_.push_back(sizeof(size_t)); else attribute_sizes_.push_back(array_schema->cell_size(attribute_ids_[i])); } subarray_ = malloc(2*coords_size_); memcpy(subarray_, array_->subarray(), 2*coords_size_); // Calculate expanded subarray expanded_subarray_ = malloc(2*coords_size_); memcpy(expanded_subarray_, subarray_, 2*coords_size_); array_schema->expand_domain(expanded_subarray_); // Calculate number of buffers calculate_buffer_num(); // Initialize tile slab info and state, and copy state init_tile_slab_info(); init_tile_slab_state(); init_copy_state(); } ArraySortedWriteState::~ArraySortedWriteState() { // Clean up free(subarray_); free(expanded_subarray_); free(tile_coords_); free(tile_domain_); for(int i=0; i<2; ++i) { free(tile_slab_[i]); free(tile_slab_norm_[i]); } // Free tile slab info and state, and copy state free_copy_state(); free_tile_slab_state(); free_tile_slab_info(); // Cancel AIO thread aio_thread_canceled_ = true; for(int i=0; i<2; ++i) release_copy(i); // Wait for thread to be destroyed while(aio_thread_running_); // Join with the terminated thread pthread_join(aio_thread_, NULL); // Destroy conditions and mutexes for(int i=0; i<2; ++i) { if(pthread_cond_destroy(&(aio_cond_[i]))) { std::string errmsg = "Cannot destroy AIO mutex condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; } if(pthread_cond_destroy(&(copy_cond_[i]))) { std::string errmsg = "Cannot destroy copy mutex condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; } } if(pthread_mutex_destroy(&aio_mtx_)) { std::string errmsg = "Cannot destroy AIO mutex"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; } if(pthread_mutex_destroy(©_mtx_)) { std::string errmsg = "Cannot destroy copy mutex"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; } } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int ArraySortedWriteState::init() { // Initialize the mutexes and conditions if(pthread_mutex_init(&aio_mtx_, NULL)) { std::string errmsg = "Cannot initialize IO mutex"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } if(pthread_mutex_init(©_mtx_, NULL)) { std::string errmsg = "Cannot initialize copy mutex"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } for(int i=0; i<2; ++i) { aio_cond_[i] = PTHREAD_COND_INITIALIZER; if(pthread_cond_init(&(aio_cond_[i]), NULL)) { std::string errmsg = "Cannot initialize IO mutex condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } copy_cond_[i] = PTHREAD_COND_INITIALIZER; if(pthread_cond_init(&(copy_cond_[i]), NULL)) { std::string errmsg = "Cannot initialize copy mutex condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } } // Initialize functors const ArraySchema* array_schema = array_->array_schema(); int mode = array_->mode(); int cell_order = array_schema->cell_order(); int tile_order = array_schema->tile_order(); int coords_type = array_schema->coords_type(); if(mode == TILEDB_ARRAY_WRITE_SORTED_ROW) { if(coords_type == TILEDB_INT32) { advance_cell_slab_ = advance_cell_slab_row_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; } else if(coords_type == TILEDB_INT64) { advance_cell_slab_ = advance_cell_slab_row_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; } else { assert(0); } } else { // mode == TILEDB_ARRAY_WRITE_SORTED_COL if(coords_type == TILEDB_INT32) { advance_cell_slab_ = advance_cell_slab_col_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; } else if(coords_type == TILEDB_INT64) { advance_cell_slab_ = advance_cell_slab_col_s; calculate_cell_slab_info_ = (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; } else { assert(0); } } if(tile_order == TILEDB_ROW_MAJOR) { if(coords_type == TILEDB_INT32) calculate_tile_slab_info_ = calculate_tile_slab_info_row; else if(coords_type == TILEDB_INT64) calculate_tile_slab_info_ = calculate_tile_slab_info_row; else assert(0); } else { // tile_order == TILEDB_COL_MAJOR if(coords_type == TILEDB_INT32) calculate_tile_slab_info_ = calculate_tile_slab_info_col; else if(coords_type == TILEDB_INT64) calculate_tile_slab_info_ = calculate_tile_slab_info_col; else assert(0); } // Create the thread that will be handling all the asynchronous IOs if(pthread_create( &aio_thread_, NULL, ArraySortedWriteState::aio_handler, this)) { std::string errmsg = "Cannot create AIO thread"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } aio_thread_running_ = true; // Success return TILEDB_ASWS_OK; } int ArraySortedWriteState::write( const void** buffers, const size_t* buffer_sizes) { // Locally store user buffer information create_user_buffers(buffers, buffer_sizes); // Create buffers if(create_copy_state_buffers() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Create AIO requests init_aio_requests(); // Call the appropriate templated read int type = array_->array_schema()->coords_type(); if(type == TILEDB_INT32) { return write(); } else if(type == TILEDB_INT64) { return write(); } else { assert(0); return TILEDB_ASWS_ERR; } // Success return TILEDB_ASWS_OK; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ template void *ArraySortedWriteState::advance_cell_slab_col_s(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int aid = ((ASWS_Data*) data)->id_; asws->advance_cell_slab_col(aid); return NULL; } template void *ArraySortedWriteState::advance_cell_slab_row_s(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int aid = ((ASWS_Data*) data)->id_; asws->advance_cell_slab_row(aid); return NULL; } template void ArraySortedWriteState::advance_cell_slab_col(int aid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; // Tile id int64_t cell_slab_num = tile_slab_info_[copy_id_].cell_slab_num_[tid]; T* current_coords = (T*) tile_slab_state_.current_coords_[aid]; const T* tile_slab = (const T*) tile_slab_norm_[copy_id_]; // Advance cell slab coordinates int d = 0; current_coords[d] += cell_slab_num; int64_t dim_overflow; for(int i=0; i tile_slab[2*(dim_num_-1)+1]) { tile_slab_state_.copy_tile_slab_done_[aid] = true; return; } // Calculate new tile and offset for the current coords update_current_tile_and_offset(aid); } template void ArraySortedWriteState::advance_cell_slab_row(int aid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; // Tile id int64_t cell_slab_num = tile_slab_info_[copy_id_].cell_slab_num_[tid]; T* current_coords = (T*) tile_slab_state_.current_coords_[aid]; const T* tile_slab = (const T*) tile_slab_norm_[copy_id_]; // Advance cell slab coordinates int d = dim_num_-1; current_coords[d] += cell_slab_num; int64_t dim_overflow; for(int i=d; i>0; --i) { dim_overflow = (current_coords[i] - tile_slab[2*i]) / (tile_slab[2*i+1]-tile_slab[2*i]+1); current_coords[i-1] += dim_overflow; current_coords[i] -= dim_overflow * (tile_slab[2*i+1]-tile_slab[2*i]+1); } // Check if done if(current_coords[0] > tile_slab[1]) { tile_slab_state_.copy_tile_slab_done_[aid] = true; return; } // Calculate new tile and offset for the current coords update_current_tile_and_offset(aid); } void *ArraySortedWriteState::aio_done(void* data) { // Retrieve data ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; // Manage the mutexes and conditions asws->release_aio(id); return NULL; } void ArraySortedWriteState::block_aio(int id) { lock_aio_mtx(); wait_aio_[id] = true; unlock_aio_mtx(); } void ArraySortedWriteState::block_copy(int id) { lock_copy_mtx(); wait_copy_[id] = true; unlock_copy_mtx(); } void ArraySortedWriteState::calculate_buffer_num() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); // Calculate number of buffers buffer_num_ = 0; int attribute_id_num = (int) attribute_ids_.size(); for(int i=0; ivar_size(attribute_ids_[i])) ++buffer_num_; else // Variable-sized attribute buffer_num_ += 2; } } template void *ArraySortedWriteState::calculate_cell_slab_info_col_col_s(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; int tid = ((ASWS_Data*) data)->id_2_; asws->calculate_cell_slab_info_col_col(id, tid); return NULL; } template void *ArraySortedWriteState::calculate_cell_slab_info_col_row_s(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; int tid = ((ASWS_Data*) data)->id_2_; asws->calculate_cell_slab_info_col_row(id, tid); return NULL; } template void *ArraySortedWriteState::calculate_cell_slab_info_row_col_s(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; int tid = ((ASWS_Data*) data)->id_2_; asws->calculate_cell_slab_info_row_col(id, tid); return NULL; } template void *ArraySortedWriteState::calculate_cell_slab_info_row_row_s(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; int tid = ((ASWS_Data*) data)->id_2_; asws->calculate_cell_slab_info_row_row(id, tid); return NULL; } template void ArraySortedWriteState::calculate_cell_slab_info_col_col( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* range_overlap = (const T*) tile_slab_info_[id].range_overlap_[tid]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); int64_t cell_num; // Calculate number of cells in cell slab cell_num = range_overlap[1] - range_overlap[0] + 1; tile_slab_info_[id].cell_slab_num_[tid] = cell_num; // Calculate size of a cell slab per attribute for(int aid=0; aid void ArraySortedWriteState::calculate_cell_slab_info_row_row( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* range_overlap = (const T*) tile_slab_info_[id].range_overlap_[tid]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); int64_t cell_num; // Calculate number of cells in cell slab cell_num = range_overlap[2*(dim_num_-1)+1] - range_overlap[2*(dim_num_-1)] +1; tile_slab_info_[id].cell_slab_num_[tid] = cell_num; // Calculate size of a cell slab per attribute for(int aid=0; aid=0; --i) { cell_offset *= tile_extents[i+1]; tile_slab_info_[id].cell_offset_per_dim_[tid][i] = cell_offset; } } template void ArraySortedWriteState::calculate_cell_slab_info_col_row( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); // Calculate number of cells in cell slab tile_slab_info_[id].cell_slab_num_[tid] = 1; // Calculate size of a cell slab per attribute for(int aid=0; aid=0; --i) { cell_offset *= tile_extents[i+1]; tile_slab_info_[id].cell_offset_per_dim_[tid][i] = cell_offset; } } template void ArraySortedWriteState::calculate_cell_slab_info_row_col( int id, int64_t tid) { // For easy reference int anum = (int) attribute_ids_.size(); const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); // Calculate number of cells in cell slab tile_slab_info_[id].cell_slab_num_[tid] = 1; // Calculate size of a cell slab per attribute for(int aid=0; aid void ArraySortedWriteState::calculate_tile_domain(int id) { // Initializations tile_coords_ = malloc(coords_size_); tile_domain_ = malloc(2*coords_size_); // For easy reference const T* tile_slab = (const T*) tile_slab_norm_[id]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); T* tile_coords = (T*) tile_coords_; T* tile_domain = (T*) tile_domain_; // Calculate tile domain and initial tile coordinates for(int i=0; i void ArraySortedWriteState::calculate_tile_slab_info(int id) { // Calculate number of tiles, if they are not already calculated if(tile_slab_info_[id].tile_num_ == -1) init_tile_slab_info(id); // Calculate tile domain, if not calculated yet if(tile_domain_ == NULL) calculate_tile_domain(id); // Reset tile coordinates reset_tile_coords(); // Calculate tile slab info ASWS_Data asws_data = { id, 0, this }; (*calculate_tile_slab_info_)(&asws_data); } template void *ArraySortedWriteState::calculate_tile_slab_info_col(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; asws->calculate_tile_slab_info_col(id); return NULL; } template void ArraySortedWriteState::calculate_tile_slab_info_col(int id) { // For easy reference const T* tile_domain = (const T*) tile_domain_; T* tile_coords = (T*) tile_coords_; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); T** range_overlap = (T**) tile_slab_info_[id].range_overlap_; const T* tile_slab = (const T*) tile_slab_norm_[id]; int64_t tile_offset, tile_cell_num; int64_t total_cell_num = 0; int anum = (int) attribute_ids_.size(); int d; // Iterate over all tiles in the tile domain int64_t tid=0; // Tile id while(tile_coords[dim_num_-1] <= tile_domain[2*(dim_num_-1)+1]) { // Calculate range overlap, number of cells in the tile tile_cell_num = 1; for(int i=0; i tile_domain[2*d+1]) { tile_coords[d] = tile_domain[2*d]; ++tile_coords[++d]; } // Advance tile id ++tid; } } template void *ArraySortedWriteState::calculate_tile_slab_info_row(void* data) { ArraySortedWriteState* asws = ((ASWS_Data*) data)->asws_; int id = ((ASWS_Data*) data)->id_; asws->calculate_tile_slab_info_row(id); return NULL; } template void ArraySortedWriteState::calculate_tile_slab_info_row(int id) { // For easy reference const T* tile_domain = (const T*) tile_domain_; T* tile_coords = (T*) tile_coords_; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); T** range_overlap = (T**) tile_slab_info_[id].range_overlap_; const T* tile_slab = (const T*) tile_slab_norm_[id]; int64_t tile_offset, tile_cell_num; int64_t total_cell_num = 0; int anum = (int) attribute_ids_.size(); int d; // Iterate over all tiles in the tile domain int64_t tid=0; // Tile id while(tile_coords[0] <= tile_domain[1]) { // Calculate range overlap, number of cells in the tile tile_cell_num = 1; for(int i=0; i=0; --i) { tile_offset *= (tile_domain[2*(i+1)+1] - tile_domain[2*(i+1)] + 1); tile_slab_info_[id].tile_offset_per_dim_[i] = tile_offset; } // Calculate cell slab info ASWS_Data asws_data = { id, tid, this }; (*calculate_cell_slab_info_)(&asws_data); // Calculate start offsets for(int aid=0; aid 0 && tile_coords[d] > tile_domain[2*d+1]) { tile_coords[d] = tile_domain[2*d]; ++tile_coords[--d]; } // Advance tile id ++tid; } } void *ArraySortedWriteState::aio_handler(void* context) { // For easy reference ArraySortedWriteState* asws = (ArraySortedWriteState*) context; // This will enter an indefinite loop that will handle all incoming copy // requests int coords_type = asws->array_->array_schema()->coords_type(); if(coords_type == TILEDB_INT32) asws->handle_aio_requests(); else if(coords_type == TILEDB_INT64) asws->handle_aio_requests(); else assert(0); // Return return NULL; } void ArraySortedWriteState::copy_tile_slab() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); // Copy tile slab for each attribute separately for(int i=0, b=0; i<(int)attribute_ids_.size(); ++i) { int type = array_schema->type(attribute_ids_[i]); if(!array_schema->var_size(attribute_ids_[i])) { if(type == TILEDB_CHAR) copy_tile_slab(i, b); else if(type == TILEDB_INT8) copy_tile_slab(i, b); else if(type == TILEDB_INT16) copy_tile_slab(i, b); else if(type == TILEDB_INT32) copy_tile_slab(i, b); else if(type == TILEDB_INT64) copy_tile_slab(i, b); else if(type == TILEDB_UINT8) copy_tile_slab(i, b); else if(type == TILEDB_UINT16) copy_tile_slab(i, b); else if(type == TILEDB_UINT32) copy_tile_slab(i, b); else if(type == TILEDB_UINT64) copy_tile_slab(i, b); else if(type == TILEDB_FLOAT32) copy_tile_slab(i, b); else if(type == TILEDB_FLOAT64) copy_tile_slab(i, b); ++b; } else { if(type == TILEDB_CHAR) copy_tile_slab_var(i, b); else if(type == TILEDB_INT8) copy_tile_slab_var(i, b); else if(type == TILEDB_INT16) copy_tile_slab_var(i, b); else if(type == TILEDB_INT32) copy_tile_slab_var(i, b); else if(type == TILEDB_INT64) copy_tile_slab_var(i, b); else if(type == TILEDB_UINT8) copy_tile_slab_var(i, b); else if(type == TILEDB_UINT16) copy_tile_slab_var(i, b); else if(type == TILEDB_UINT32) copy_tile_slab_var(i, b); else if(type == TILEDB_UINT64) copy_tile_slab_var(i, b); else if(type == TILEDB_FLOAT32) copy_tile_slab_var(i, b); else if(type == TILEDB_FLOAT64) copy_tile_slab_var(i, b); b += 2; } } } template void ArraySortedWriteState::copy_tile_slab(int aid, int bid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; size_t& buffer_offset = buffer_offsets_[bid]; char* buffer = (char*) buffers_[bid]; char* local_buffer = (char*) copy_state_.buffers_[copy_id_][bid]; size_t& local_buffer_offset = copy_state_.buffer_offsets_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; ASWS_Data asws_data = { aid, bid, this }; // Fill with empty fill_with_empty(bid); // Important for initializing the current tile and offsets! update_current_tile_and_offset(aid); // Iterate over the tile slab cells for(;;) { // For easy reference size_t cell_slab_size = tile_slab_info_[copy_id_].cell_slab_size_[aid][tid]; size_t& local_buffer_offset_cur = tile_slab_state_.current_offsets_[aid]; // Copy cell slab from user to local buffer memcpy( local_buffer + local_buffer_offset_cur, buffer + buffer_offset, cell_slab_size); // Update user buffer offset buffer_offset += cell_slab_size; // Prepare for new slab (*advance_cell_slab_)(&asws_data); // Terminating condition if(tile_slab_state_.copy_tile_slab_done_[aid]) break; } // Set local buffer offset local_buffer_offset = local_buffer_size; } template void ArraySortedWriteState::copy_tile_slab_var(int aid, int bid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; size_t& buffer_offset = buffer_offsets_[bid]; size_t buffer_size = buffer_sizes_[bid]; size_t buffer_var_size = buffer_sizes_[bid+1]; char* buffer_var = (char*) buffers_[bid+1]; size_t* buffer_s = (size_t*) buffers_[bid]; char* local_buffer = (char*) copy_state_.buffers_[copy_id_][bid]; size_t* local_buffer_s = (size_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; size_t& local_buffer_offset = copy_state_.buffer_offsets_[copy_id_][bid]; char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t& local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; size_t& local_buffer_var_size = copy_state_.buffer_sizes_[copy_id_][bid+1]; int64_t cell_num_in_buffer = buffer_size / sizeof(size_t); int64_t cell_num_in_tile_slab = local_buffer_size / sizeof(size_t); ASWS_Data asws_data = { aid, 0, this }; // Important for initializing the current tile and offsets! update_current_tile_and_offset(aid); // Fill the local buffer offsets with zeros bzero(local_buffer, local_buffer_size); // Handle offsets first for(;;) { // For easy reference size_t cell_slab_size = tile_slab_info_[copy_id_].cell_slab_size_[aid][tid]; int64_t cell_num_in_slab = cell_slab_size / sizeof(size_t); size_t local_buffer_offset_cur = tile_slab_state_.current_offsets_[aid]; // Calculate variable cell slab size int64_t cell_start = buffer_offset / sizeof(size_t); int64_t cell_end = cell_start + cell_num_in_slab; // Keep track of where each variable-sized cell is. // Note that cell ids start with 1 here! for(int64_t i=cell_start+1; i<=cell_end; ++i) { memcpy( local_buffer + local_buffer_offset_cur, &i, sizeof(size_t)); local_buffer_offset_cur += sizeof(size_t); buffer_offset += sizeof(size_t); } // Prepare for new slab (*advance_cell_slab_)(&asws_data); // Terminating condition if(tile_slab_state_.copy_tile_slab_done_[aid]) break; } // Rectify offsets and copy variable-sized cells int64_t cell; size_t cell_size_var; for(int i=0; i(bid); local_buffer_offset_var += sizeof(T); continue; } // Find size of variable-sized cell cell = local_buffer_s[i]-1; // So that cell ids start from 0 cell_size_var = (cell == cell_num_in_buffer-1) ? buffer_var_size - buffer_s[cell] : buffer_s[cell+1] - buffer_s[cell]; // Rectify offset local_buffer_s[i] = local_buffer_offset_var; // Expand the variable-sized buffer if necessary while(local_buffer_offset_var + cell_size_var > local_buffer_var_size) { expand_buffer( copy_state_.buffers_[copy_id_][bid+1], copy_state_.buffer_sizes_[copy_id_][bid+1]); local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; } // Copy variable-sized cell memcpy( local_buffer_var + local_buffer_offset_var, buffer_var + buffer_s[cell], cell_size_var); local_buffer_offset_var += cell_size_var; } // Set local buffer offset local_buffer_offset = local_buffer_size; } int ArraySortedWriteState::create_copy_state_buffers() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); // Get cell number in a (full) tile slab int64_t tile_slab_cell_num; if(array_->mode() == TILEDB_ARRAY_WRITE_SORTED_ROW) tile_slab_cell_num = array_schema->tile_slab_row_cell_num(expanded_subarray_); else // TILEDB_ARRAY_WRITE_SORTED_COL tile_slab_cell_num = array_schema->tile_slab_col_cell_num(expanded_subarray_); // Calculate buffer sizes int attribute_id_num = (int) attribute_ids_.size(); for(int j=0; j<2; ++j) { copy_state_.buffer_sizes_[j] = new size_t[buffer_num_]; for(int i=0, b=0; ivar_size(attribute_ids_[i])) { copy_state_.buffer_sizes_[j][b++] = tile_slab_cell_num * array_schema->cell_size(attribute_ids_[i]); } else { // Variable-sized attribute copy_state_.buffer_sizes_[j][b++] = tile_slab_cell_num * sizeof(size_t); copy_state_.buffer_sizes_[j][b++] = 2*tile_slab_cell_num*sizeof(size_t); } } } // Allocate buffers for(int j=0; j<2; ++j) { copy_state_.buffers_[j] = (void**) malloc(buffer_num_ * sizeof(void*)); if(copy_state_.buffers_[j] == NULL) { std::string errmsg = "Cannot create local buffers"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } for(int b=0; b < buffer_num_; ++b) { copy_state_.buffers_[j][b] = malloc(copy_state_.buffer_sizes_[j][b]); if(copy_state_.buffers_[j][b] == NULL) { std::string errmsg = "Cannot allocate local buffer"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } } } // Success return TILEDB_ASWS_OK; } void ArraySortedWriteState::create_user_buffers( const void** buffers, const size_t* buffer_sizes) { buffers_ = buffers; buffer_sizes_ = buffer_sizes; buffer_offsets_ = new size_t[buffer_num_]; for(int i=0; i void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference char* local_buffer = (char*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; char empty = TILEDB_EMPTY_CHAR; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(char), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference int8_t* local_buffer = (int8_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; int8_t empty = TILEDB_EMPTY_INT8; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(int8_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference int16_t* local_buffer = (int16_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; int16_t empty = TILEDB_EMPTY_INT16; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(int16_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference int32_t* local_buffer = (int32_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; int32_t empty = TILEDB_EMPTY_INT32; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(int32_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference int64_t* local_buffer = (int64_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; int64_t empty = TILEDB_EMPTY_INT64; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(int64_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference uint8_t* local_buffer = (uint8_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; uint8_t empty = TILEDB_EMPTY_UINT8; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint8_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference uint16_t* local_buffer = (uint16_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; uint16_t empty = TILEDB_EMPTY_UINT16; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint16_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference uint32_t* local_buffer = (uint32_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; uint32_t empty = TILEDB_EMPTY_UINT32; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint32_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference uint64_t* local_buffer = (uint64_t*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; uint64_t empty = TILEDB_EMPTY_UINT64; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint64_t), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference float* local_buffer = (float*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; float empty = TILEDB_EMPTY_FLOAT32; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(float), ++i) local_buffer[i] = empty; } template<> void ArraySortedWriteState::fill_with_empty(int bid) { // For easy reference double* local_buffer = (double*) copy_state_.buffers_[copy_id_][bid]; size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; double empty = TILEDB_EMPTY_FLOAT64; // Fill with empty values size_t offset = 0; for(int64_t i=0; offset < local_buffer_size; offset += sizeof(double), ++i) local_buffer[i] = empty; } // Specializations for fill_with_empty_var template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; char empty = TILEDB_EMPTY_CHAR; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(char)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int empty = TILEDB_EMPTY_INT8; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(int8_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int64_t empty = TILEDB_EMPTY_INT16; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(int16_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int empty = TILEDB_EMPTY_INT32; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(int32_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int64_t empty = TILEDB_EMPTY_INT64; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(int64_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int empty = TILEDB_EMPTY_UINT8; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint8_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int64_t empty = TILEDB_EMPTY_UINT16; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint16_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int empty = TILEDB_EMPTY_UINT32; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint32_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; int64_t empty = TILEDB_EMPTY_UINT64; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint64_t)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; float empty = TILEDB_EMPTY_FLOAT32; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(float)); } template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; double empty = TILEDB_EMPTY_FLOAT64; // Fill an empty value memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(double)); } void ArraySortedWriteState::free_copy_state() { for(int i=0; i<2; ++i) { if(copy_state_.buffer_sizes_[i] != NULL) delete [] copy_state_.buffer_sizes_[i]; if(copy_state_.buffers_[i] != NULL) { for(int b=0; b int64_t ArraySortedWriteState::get_cell_id(int aid) { // For easy reference const T* current_coords = (const T*) tile_slab_state_.current_coords_[aid]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); int64_t tid = tile_slab_state_.current_tile_[aid]; int64_t* cell_offset_per_dim = tile_slab_info_[copy_id_].cell_offset_per_dim_[tid]; // Calculate cell id int64_t cid = 0; for(int i=0; i int64_t ArraySortedWriteState::get_tile_id(int aid) { // For easy reference const T* current_coords = (const T*) tile_slab_state_.current_coords_[aid]; const T* tile_extents = (const T*) array_->array_schema()->tile_extents(); int64_t* tile_offset_per_dim = tile_slab_info_[copy_id_].tile_offset_per_dim_; // Calculate tile id int64_t tid = 0; for(int i=0; i void ArraySortedWriteState::handle_aio_requests() { // Handle AIO requests indefinitely for(;;) { // Wait for AIO wait_copy(aio_id_); // Kill thread if(aio_thread_canceled_) { aio_thread_running_ = false; return; } // Block copy block_copy(aio_id_); // Send AIO request send_aio_request(aio_id_); // Advance AIO id aio_id_ = (aio_id_ + 1) % 2; } } void ArraySortedWriteState::init_aio_requests() { // For easy reference int mode = array_->mode(); int tile_order = array_->array_schema()->tile_order(); const void* subarray = array_->subarray(); bool separate_fragments = (mode == TILEDB_ARRAY_WRITE_SORTED_COL && tile_order == TILEDB_ROW_MAJOR) || (mode == TILEDB_ARRAY_WRITE_SORTED_ROW && tile_order == TILEDB_COL_MAJOR); // Initialize AIO requests for(int i=0; i<2; ++i) { aio_data_[i] = { i, 0, this }; memset(&aio_request_[i], 0, sizeof(AIO_Request)); aio_request_[i].id_ = (separate_fragments) ? aio_cnt_++ : 0; aio_request_[i].buffer_sizes_ = copy_state_.buffer_offsets_[i]; aio_request_[i].buffers_ = copy_state_.buffers_[i]; aio_request_[i].mode_ = TILEDB_ARRAY_WRITE; aio_request_[i].subarray_ = (separate_fragments) ? tile_slab_[i] : subarray; aio_request_[i].completion_handle_ = aio_done; aio_request_[i].completion_data_ = &(aio_data_[i]); aio_request_[i].overflow_ = NULL; aio_request_[i].status_ = &(aio_status_[i]); } } void ArraySortedWriteState::init_copy_state() { for(int j=0; j<2; ++j) { copy_state_.buffer_offsets_[j] = new size_t[buffer_num_]; copy_state_.buffer_sizes_[j] = new size_t[buffer_num_]; copy_state_.buffers_[j] = new void*[buffer_num_]; for(int i=0; i void ArraySortedWriteState::init_tile_slab_info(int id) { // Sanity check assert(array_->array_schema()->dense()); // For easy reference int anum = (int) attribute_ids_.size(); // Calculate tile number int64_t tile_num = array_->array_schema()->tile_num(tile_slab_[id]); // Initializations tile_slab_info_[id].cell_offset_per_dim_ = new int64_t*[tile_num]; tile_slab_info_[id].cell_slab_num_ = new int64_t[tile_num]; tile_slab_info_[id].range_overlap_ = new void*[tile_num]; for(int64_t i=0; i bool ArraySortedWriteState::next_tile_slab_col() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); const T* domain = static_cast(array_schema->domain()); const T* tile_extents = static_cast(array_schema->tile_extents()); T* tile_slab[2]; T* tile_slab_norm = static_cast(tile_slab_norm_[copy_id_]); for(int i=0; i<2; ++i) tile_slab[i] = static_cast(tile_slab_[i]); int prev_id = (copy_id_+1)%2; T tile_start; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][2*(dim_num_-1) + 1] == subarray[2*(dim_num_-1) + 1]) { return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[copy_id_][2*(dim_num_-1)] = subarray[2*(dim_num_-1)]; T upper = subarray[2*(dim_num_-1)] + tile_extents[dim_num_-1]; T cropped_upper = (upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1] * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[copy_id_][2*(dim_num_-1)+1] = std::min(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i(copy_id_); // Mark this tile slab as initialized tile_slab_init_[copy_id_] = true; // Success return true; } template bool ArraySortedWriteState::next_tile_slab_row() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); const T* domain = static_cast(array_schema->domain()); const T* tile_extents = static_cast(array_schema->tile_extents()); T* tile_slab[2]; T* tile_slab_norm = static_cast(tile_slab_norm_[copy_id_]); for(int i=0; i<2; ++i) tile_slab[i] = static_cast(tile_slab_[i]); int prev_id = (copy_id_+1)%2; T tile_start; // Check again if done, this time based on the tile slab and subarray if(tile_slab_init_[prev_id] && tile_slab[prev_id][1] == subarray[1]) { return false; } // If this is the first time this function is called, initialize if(!tile_slab_init_[prev_id]) { // Crop the subarray extent along the first axis to fit in the first tile tile_slab[copy_id_][0] = subarray[0]; T upper = subarray[0] + tile_extents[0]; T cropped_upper = (upper - domain[0]) / tile_extents[0] * tile_extents[0] + domain[0]; tile_slab[copy_id_][1] = std::min(cropped_upper - 1, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i(copy_id_); // Mark this tile slab as initialized tile_slab_init_[copy_id_] = true; // Success return true; } template int ArraySortedWriteState::write() { // For easy reference int mode = array_->mode(); if(mode == TILEDB_ARRAY_WRITE_SORTED_COL) { return write_sorted_col(); } else if(mode == TILEDB_ARRAY_WRITE_SORTED_ROW) { return write_sorted_row(); } else { assert(0); // The code should never reach here return TILEDB_ASWS_ERR; } } template int ArraySortedWriteState::write_sorted_col() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); // Check if this can be satisfied with a default write if(array_schema->cell_order() == TILEDB_COL_MAJOR && !memcmp(subarray_, expanded_subarray_, 2*coords_size_) && array_schema->is_contained_in_tile_slab_row(subarray)) return array_->write_default(buffers_, buffer_sizes_); // Iterate over each tile slab while(next_tile_slab_col()) { // Wait for AIO wait_aio(copy_id_); // Block AIO block_aio(copy_id_); // Reset the tile slab state and copy state reset_tile_slab_state(); reset_copy_state(); // Copy tile slab copy_tile_slab(); // Release copy release_copy(copy_id_); // Advance copy id copy_id_ = (copy_id_ + 1) % 2; } // Wait for last AIO to finish wait_aio((copy_id_ + 1) % 2); // The following will make the AIO thread terminate aio_thread_canceled_ = true; release_copy(copy_id_); // Success return TILEDB_ASWS_OK; } template int ArraySortedWriteState::write_sorted_row() { // For easy reference const ArraySchema* array_schema = array_->array_schema(); const T* subarray = static_cast(subarray_); // Check if this can be satisfied with a default write if(array_schema->cell_order() == TILEDB_ROW_MAJOR && !memcmp(subarray_, expanded_subarray_, 2*coords_size_) && array_schema->is_contained_in_tile_slab_col(subarray)) return array_->write_default(buffers_, buffer_sizes_); // Iterate over each tile slab while(next_tile_slab_row()) { // Wait for AIO wait_aio(copy_id_); // Block AIO block_aio(copy_id_); // Reset the tile slab state reset_tile_slab_state(); reset_copy_state(); // Copy tile slab copy_tile_slab(); // Release copy release_copy(copy_id_); // Advance copy id copy_id_ = (copy_id_ + 1) % 2; } // Wait for last AIO to finish wait_aio((copy_id_ + 1) % 2); // The following will make the AIO thread terminate aio_thread_canceled_ = true; release_copy(copy_id_); // Success return TILEDB_ASWS_OK; } int ArraySortedWriteState::release_aio(int id) { // Lock the AIO mutex if(lock_aio_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Set AIO flag wait_aio_[id] = false; // Signal condition if(pthread_cond_signal(&(aio_cond_[id]))) { std::string errmsg = "Cannot signal AIO condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } // Unlock the AIO mutex if(unlock_aio_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Success return TILEDB_ASWS_OK; } int ArraySortedWriteState::release_copy(int id) { // Lock the copy mutex if(lock_copy_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Set copy flag wait_copy_[id] = false; // Signal condition if(pthread_cond_signal(©_cond_[id])) { std::string errmsg = "Cannot signal copy condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } // Unlock the copy mutex if(unlock_copy_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Success return TILEDB_ASWS_OK; } void ArraySortedWriteState::reset_copy_state() { for(int i=0; i void ArraySortedWriteState::reset_tile_coords() { T* tile_coords = (T*) tile_coords_; for(int i=0; i void ArraySortedWriteState::reset_tile_slab_state() { // For easy reference int anum = (int) attribute_ids_.size(); T** current_coords = (T**) tile_slab_state_.current_coords_; const T* tile_slab = (const T*) tile_slab_norm_[copy_id_]; // Reset values for(int i=0; iarray_clone(); // Sanity check assert(array_clone != NULL); // Send the AIO request to the clone array if(array_clone->aio_write(&(aio_request_[aio_id])) != TILEDB_AR_OK) { tiledb_asws_errmsg = tiledb_ar_errmsg; return TILEDB_ASWS_ERR; } // Success return TILEDB_ASWS_OK; } int ArraySortedWriteState::unlock_aio_mtx() { if(pthread_mutex_unlock(&aio_mtx_)) { std::string errmsg = "Cannot unlock AIO mutex"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } // Success return TILEDB_ASWS_OK; } int ArraySortedWriteState::unlock_copy_mtx() { if(pthread_mutex_unlock(©_mtx_)) { std::string errmsg = "Cannot unlock copy mutex"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } // Success return TILEDB_ASWS_OK; } void ArraySortedWriteState::update_current_tile_and_offset(int aid) { // For easy reference int coords_type = array_->array_schema()->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) update_current_tile_and_offset(aid); else if(coords_type == TILEDB_INT64) update_current_tile_and_offset(aid); else if(coords_type == TILEDB_FLOAT32) update_current_tile_and_offset(aid); else if(coords_type == TILEDB_FLOAT64) update_current_tile_and_offset(aid); else assert(0); } template void ArraySortedWriteState::update_current_tile_and_offset(int aid) { // For easy reference int64_t& tid = tile_slab_state_.current_tile_[aid]; size_t& current_offset = tile_slab_state_.current_offsets_[aid]; int64_t cid; // Calculate the new tile id tid = get_tile_id(aid); // Calculate the cell id cid = get_cell_id(aid); // Calculate new offset current_offset = tile_slab_info_[copy_id_].start_offsets_[aid][tid] + cid * attribute_sizes_[aid]; } int ArraySortedWriteState::wait_aio(int id) { // Lock AIO mutex if(lock_aio_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Wait to be signaled while(wait_aio_[id]) { if(pthread_cond_wait(&(aio_cond_[id]), &aio_mtx_)) { std::string errmsg = "Cannot wait on IO mutex condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } } // Unlock AIO mutex if(unlock_aio_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Success return TILEDB_ASWS_OK; } int ArraySortedWriteState::wait_copy(int id) { // Lock copy mutex if(lock_copy_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Wait to be signaled while(wait_copy_[id]) { if(pthread_cond_wait(&(copy_cond_[id]), ©_mtx_)) { std::string errmsg = "Cannot wait on copy mutex condition"; PRINT_ERROR(errmsg); tiledb_asws_errmsg = TILEDB_ASWS_ERRMSG + errmsg; return TILEDB_ASWS_ERR; } } // Unlock copy mutex if(unlock_copy_mtx() != TILEDB_ASWS_OK) return TILEDB_ASWS_ERR; // Success return TILEDB_ASWS_OK; } // Explicit template instantiations template int ArraySortedWriteState::write_sorted_col(); template int ArraySortedWriteState::write_sorted_col(); template int ArraySortedWriteState::write_sorted_row(); template int ArraySortedWriteState::write_sorted_row(); genomicsdb-0.0~git20231212.9d7ddd0/core/src/c_api/000077500000000000000000000000001453617025200211375ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/c_api/tiledb.cc000066400000000000000000001576431453617025200227310ustar00rootroot00000000000000/** * @file tiledb.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corp. * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file defines the C API of TileDB. */ #include "aio_request.h" #include "tiledb.h" #include "tiledb_utils.h" #include "array_schema_c.h" #include "storage_manager.h" #include "storage_manager_config.h" #include "utils.h" #include "trace.h" #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ char tiledb_errmsg[TILEDB_ERRMSG_MAX_LEN]; /* ****************************** */ /* CONTEXT */ /* ****************************** */ typedef struct TileDB_CTX { StorageManager* storage_manager_; } TileDB_CTX; int tiledb_ctx_init( TileDB_CTX** tiledb_ctx, const TileDB_Config* tiledb_config) { if (tiledb_config && tiledb_config->home_) { TRACE_FN_ARG("Home=" << tiledb_config->home_); std::string home = std::string(tiledb_config->home_, strlen(tiledb_config->home_)); if (TileDBUtils::is_cloud_path(home) && !is_supported_cloud_path(home)) { std::string errmsg = "No TileDB support for URL=" + home; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, errmsg.c_str()); return TILEDB_ERR; } } // Initialize error message to empty strcpy(tiledb_errmsg, ""); // Initialize context *tiledb_ctx = (TileDB_CTX*) malloc(sizeof(struct TileDB_CTX)); if(*tiledb_ctx == NULL) { std::string errmsg = "Cannot initialize TileDB context; Failed to allocate memory " "space for the context"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } memset(*tiledb_ctx, 0, sizeof(struct TileDB_CTX)); // Initialize a Config object StorageManagerConfig* storage_manager_config = new StorageManagerConfig(); if(tiledb_config != NULL) { if (storage_manager_config->init( tiledb_config->home_, #ifdef HAVE_MPI tiledb_config->mpi_comm_, #endif tiledb_config->read_method_, tiledb_config->write_method_, tiledb_config->enable_shared_posixfs_optimizations_) == TILEDB_SMC_ERR) { delete storage_manager_config; free(*tiledb_ctx); *tiledb_ctx = NULL; strcpy(tiledb_errmsg, tiledb_smc_errmsg.c_str()); return TILEDB_ERR; } } // Create storage manager StorageManager* storage_manager = new StorageManager(); if(storage_manager->init(storage_manager_config) != TILEDB_SM_OK) { // No need to delete storage_manager_config as StorageManager owns it at this point // See StorageManager::config_set(). delete storage_manager; free(*tiledb_ctx); *tiledb_ctx = NULL; strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } (*tiledb_ctx)->storage_manager_ = storage_manager; // Success return TILEDB_OK; } int tiledb_ctx_finalize(TileDB_CTX* tiledb_ctx) { // Trivial case if(tiledb_ctx == NULL) return TILEDB_OK; // Finalize storage manager int rc = TILEDB_OK; if(tiledb_ctx->storage_manager_ != NULL) { rc = tiledb_ctx->storage_manager_->finalize(); delete tiledb_ctx->storage_manager_; } // Clean up free(tiledb_ctx); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } /* ****************************** */ /* SANITY CHECKS */ /* ****************************** */ inline bool sanity_check(const TileDB_CTX* tiledb_ctx) { if(tiledb_ctx == NULL || tiledb_ctx->storage_manager_ == NULL) { std::string errmsg = "Invalid TileDB context"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } else { return true; } } inline bool sanity_check(const TileDB_Array* tiledb_array) { if(tiledb_array == NULL) { std::string errmsg = "Invalid TileDB array"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } else { return true; } } inline bool sanity_check(const TileDB_ArrayIterator* tiledb_array_it) { if(tiledb_array_it == NULL) { std::string errmsg = "Invalid TileDB array iterator"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } else { return true; } } inline bool sanity_check(const TileDB_Metadata* tiledb_metadata) { if(tiledb_metadata == NULL) { std::string errmsg = "Invalid TileDB metadata"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } else { return true; } } inline bool sanity_check(const TileDB_MetadataIterator* tiledb_metadata_it) { if(tiledb_metadata_it == NULL) { std::string errmsg = "Invalid TileDB metadata iterator"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } else { return true; } } /* ****************************** */ /* WORKSPACE */ /* ****************************** */ int tiledb_workspace_create( const TileDB_CTX* tiledb_ctx, const char* workspace) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check workspace name length if(workspace == NULL || strlen(workspace) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid workspace name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Create the workspace if(tiledb_ctx->storage_manager_->workspace_create(workspace) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } /* ****************************** */ /* GROUP */ /* ****************************** */ int tiledb_group_create( const TileDB_CTX* tiledb_ctx, const char* group) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check group name length if(group == NULL || strlen(group) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid group name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Create the group if(tiledb_ctx->storage_manager_->group_create(group) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } /* ****************************** */ /* ARRAY */ /* ****************************** */ typedef struct TileDB_Array { Array* array_; const TileDB_CTX* tiledb_ctx_; } TileDB_Array; #ifdef ENABLE_MUPARSERX_EXPRESSIONS typedef struct TileDB_Expression { Expression* expression_; } TileDB_Expression; #endif int tiledb_array_set_schema( TileDB_ArraySchema* tiledb_array_schema, const char* array_name, const char** attributes, int attribute_num, int64_t capacity, int cell_order, const int* cell_val_num, const int* compression, const int* compression_level, const int* offsets_compression, const int* offsets_compression_level, int dense, const char** dimensions, int dim_num, const void* domain, size_t domain_len, const void* tile_extents, size_t tile_extents_len, int tile_order, const int* types) { // Sanity check if(tiledb_array_schema == NULL) { std::string errmsg = "Invalid array schema pointer"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } memset(tiledb_array_schema, 0, sizeof(TileDB_ArraySchema)); // Set array name size_t array_name_len = strlen(array_name); if(array_name == NULL || array_name_len > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid array name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } tiledb_array_schema->array_name_ = (char*) malloc(array_name_len+1); strcpy(tiledb_array_schema->array_name_, array_name); // Set attributes and number of attributes tiledb_array_schema->attribute_num_ = attribute_num; tiledb_array_schema->attributes_ = (char**) malloc(attribute_num*sizeof(char*)); for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } tiledb_array_schema->attributes_[i] = (char*) malloc(attribute_len+1); strcpy(tiledb_array_schema->attributes_[i], attributes[i]); } // Set dimensions tiledb_array_schema->dim_num_ = dim_num; tiledb_array_schema->dimensions_ = (char**) malloc(dim_num*sizeof(char*)); for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } tiledb_array_schema->dimensions_[i] = (char*) malloc(dimension_len+1); strcpy(tiledb_array_schema->dimensions_[i], dimensions[i]); } // Set dense tiledb_array_schema->dense_ = dense; // Set domain tiledb_array_schema->domain_ = malloc(domain_len); memcpy(tiledb_array_schema->domain_, domain, domain_len); // Set tile extents if(tile_extents != NULL) { tiledb_array_schema->tile_extents_ = malloc(tile_extents_len); memcpy(tiledb_array_schema->tile_extents_, tile_extents, tile_extents_len); } // Set types tiledb_array_schema->types_ = (int*) malloc((attribute_num+1)*sizeof(int)); for(int i=0; itypes_[i] = types[i]; } // Set cell val num if(cell_val_num != NULL) { tiledb_array_schema->cell_val_num_ = (int*) malloc((attribute_num)*sizeof(int)); for(int i=0; icell_val_num_[i] = cell_val_num[i]; } } // Set cell and tile order tiledb_array_schema->cell_order_ = cell_order; tiledb_array_schema->tile_order_ = tile_order; // Set capacity tiledb_array_schema->capacity_ = capacity; // Set compression if(compression != NULL) { tiledb_array_schema->compression_ = (int*) malloc((attribute_num+1)*sizeof(int)); for(int i=0; icompression_[i] = compression[i]; } } // Set compression levels if (compression_level != NULL) { tiledb_array_schema->compression_level_ = (int*) malloc((attribute_num+1)*sizeof(int)); for(int i=0; icompression_level_[i] = compression_level[i]; } } // Set offsets compression if(offsets_compression != NULL) { tiledb_array_schema->offsets_compression_ = (int*) malloc((attribute_num)*sizeof(int)); for(int i=0; ioffsets_compression_[i] = offsets_compression[i]; } } // Set offsets compression levels if (offsets_compression_level != NULL) { tiledb_array_schema->offsets_compression_level_ = (int*) malloc((attribute_num)*sizeof(int)); for(int i=0; ioffsets_compression_level_[i] = offsets_compression_level[i]; } } // Success return TILEDB_OK; } int tiledb_array_create( const TileDB_CTX* tiledb_ctx, const TileDB_ArraySchema* array_schema) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Copy array schema to a C struct ArraySchemaC array_schema_c; memset(&array_schema_c, 0, sizeof(array_schema_c)); array_schema_c.array_name_ = array_schema->array_name_; array_schema_c.attributes_ = array_schema->attributes_; array_schema_c.attribute_num_ = array_schema->attribute_num_; array_schema_c.capacity_ = array_schema->capacity_; array_schema_c.cell_order_ = array_schema->cell_order_; array_schema_c.cell_val_num_ = array_schema->cell_val_num_; array_schema_c.compression_ = array_schema->compression_; array_schema_c.compression_level_ = array_schema->compression_level_; array_schema_c.offsets_compression_ = array_schema->offsets_compression_; array_schema_c.offsets_compression_level_ = array_schema->offsets_compression_level_; array_schema_c.dense_ = array_schema->dense_; array_schema_c.dimensions_ = array_schema->dimensions_; array_schema_c.dim_num_ = array_schema->dim_num_; array_schema_c.domain_ = array_schema->domain_; array_schema_c.tile_extents_ = array_schema->tile_extents_; array_schema_c.tile_order_ = array_schema->tile_order_; array_schema_c.types_ = array_schema->types_; // Create the array if(tiledb_ctx->storage_manager_->array_create(&array_schema_c) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_init( const TileDB_CTX* tiledb_ctx, TileDB_Array** tiledb_array, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check array name length if(array == NULL || strlen(array) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid array name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Allocate memory for the array struct *tiledb_array = (TileDB_Array*) malloc(sizeof(struct TileDB_Array)); // Set TileDB context (*tiledb_array)->tiledb_ctx_ = tiledb_ctx; // Init the array int rc = tiledb_ctx->storage_manager_->array_init( (*tiledb_array)->array_, array, mode, subarray, attributes, attribute_num); // Return if(rc != TILEDB_SM_OK) { free(*tiledb_array); strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } else { return TILEDB_OK; } } int tiledb_array_apply_filter( const TileDB_Array* tiledb_array, const char* filter_expression) { // Apply filter if (tiledb_array->array_->apply_filter(filter_expression) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_reset_subarray( const TileDB_Array* tiledb_array, const void* subarray) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Reset subarray if(tiledb_array->array_->reset_subarray(subarray) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_reset_attributes( const TileDB_Array* tiledb_array, const char** attributes, int attribute_num) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Re-Init the array if(tiledb_array->array_->reset_attributes(attributes, attribute_num) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_get_schema( const TileDB_Array* tiledb_array, TileDB_ArraySchema* tiledb_array_schema) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Get the array schema ArraySchemaC array_schema_c; tiledb_array->array_->array_schema()->array_schema_export(&array_schema_c); // Copy the array schema C struct to the output tiledb_array_schema->array_name_ = array_schema_c.array_name_; tiledb_array_schema->attributes_ = array_schema_c.attributes_; tiledb_array_schema->attribute_num_ = array_schema_c.attribute_num_; tiledb_array_schema->capacity_ = array_schema_c.capacity_; tiledb_array_schema->cell_order_ = array_schema_c.cell_order_; tiledb_array_schema->cell_val_num_ = array_schema_c.cell_val_num_; tiledb_array_schema->compression_ = array_schema_c.compression_; tiledb_array_schema->compression_level_ = array_schema_c.compression_level_; tiledb_array_schema->offsets_compression_ = array_schema_c.offsets_compression_; tiledb_array_schema->offsets_compression_level_ = array_schema_c.offsets_compression_level_; tiledb_array_schema->dense_ = array_schema_c.dense_; tiledb_array_schema->dimensions_ = array_schema_c.dimensions_; tiledb_array_schema->dim_num_ = array_schema_c.dim_num_; tiledb_array_schema->domain_ = array_schema_c.domain_; tiledb_array_schema->tile_extents_ = array_schema_c.tile_extents_; tiledb_array_schema->tile_order_ = array_schema_c.tile_order_; tiledb_array_schema->types_ = array_schema_c.types_; // Success return TILEDB_OK; } int tiledb_array_load_schema( const TileDB_CTX* tiledb_ctx, const char* array, TileDB_ArraySchema* tiledb_array_schema) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check array name length if(array == NULL || strlen(array) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid array name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Get the array schema ArraySchema* array_schema; if(tiledb_ctx->storage_manager_->array_load_schema(array, array_schema) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } ArraySchemaC array_schema_c; array_schema->array_schema_export(&array_schema_c); // Copy the array schema C struct to the output tiledb_array_schema->array_workspace_ = array_schema_c.array_workspace_; tiledb_array_schema->array_name_ = array_schema_c.array_name_; tiledb_array_schema->attributes_ = array_schema_c.attributes_; tiledb_array_schema->attribute_num_ = array_schema_c.attribute_num_; tiledb_array_schema->capacity_ = array_schema_c.capacity_; tiledb_array_schema->cell_order_ = array_schema_c.cell_order_; tiledb_array_schema->cell_val_num_ = array_schema_c.cell_val_num_; tiledb_array_schema->compression_ = array_schema_c.compression_; tiledb_array_schema->compression_level_ = array_schema_c.compression_level_; tiledb_array_schema->offsets_compression_ = array_schema_c.offsets_compression_; tiledb_array_schema->offsets_compression_level_ = array_schema_c.offsets_compression_level_; tiledb_array_schema->dense_ = array_schema_c.dense_; tiledb_array_schema->dimensions_ = array_schema_c.dimensions_; tiledb_array_schema->dim_num_ = array_schema_c.dim_num_; tiledb_array_schema->domain_ = array_schema_c.domain_; tiledb_array_schema->tile_extents_ = array_schema_c.tile_extents_; tiledb_array_schema->tile_order_ = array_schema_c.tile_order_; tiledb_array_schema->types_ = array_schema_c.types_; // Clean up delete array_schema; // Success return TILEDB_OK; } int tiledb_array_free_schema( TileDB_ArraySchema* tiledb_array_schema) { // Trivial case if(tiledb_array_schema == NULL) return TILEDB_OK; // Free workspace if(tiledb_array_schema->array_workspace_ != NULL) free(tiledb_array_schema->array_workspace_); // Free array name if(tiledb_array_schema->array_name_ != NULL) free(tiledb_array_schema->array_name_); // Free attributes if(tiledb_array_schema->attributes_ != NULL) { for(int i=0; iattribute_num_; ++i) if(tiledb_array_schema->attributes_[i] != NULL) free(tiledb_array_schema->attributes_[i]); free(tiledb_array_schema->attributes_); } // Free dimensions if(tiledb_array_schema->dimensions_ != NULL) { for(int i=0; idim_num_; ++i) if(tiledb_array_schema->dimensions_[i] != NULL) free(tiledb_array_schema->dimensions_[i]); free(tiledb_array_schema->dimensions_); } // Free domain if(tiledb_array_schema->domain_ != NULL) free(tiledb_array_schema->domain_); // Free tile extents if(tiledb_array_schema->tile_extents_ != NULL) free(tiledb_array_schema->tile_extents_); // Free types if(tiledb_array_schema->types_ != NULL) free(tiledb_array_schema->types_); // Free compression if(tiledb_array_schema->compression_ != NULL) free(tiledb_array_schema->compression_); // Free compression level if(tiledb_array_schema->compression_level_ != NULL) free(tiledb_array_schema->compression_level_); // Free offsets compression if(tiledb_array_schema->offsets_compression_ != NULL) free(tiledb_array_schema->offsets_compression_); // Free offsets compression level if(tiledb_array_schema->compression_level_ != NULL) free(tiledb_array_schema->offsets_compression_level_); // Free cell val num if(tiledb_array_schema->cell_val_num_ != NULL) free(tiledb_array_schema->cell_val_num_); memset(tiledb_array_schema, 0, sizeof(TileDB_ArraySchema)); // Success return TILEDB_OK; } int tiledb_array_write( const TileDB_Array* tiledb_array, const void** buffers, const size_t* buffer_sizes) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Write if(tiledb_array->array_->write(buffers, buffer_sizes) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_read( const TileDB_Array* tiledb_array, void** buffers, size_t* buffer_sizes) { return tiledb_array_skip_and_read(tiledb_array, buffers, buffer_sizes, 0); //no skip counts } int tiledb_array_skip_and_read( const TileDB_Array* tiledb_array, void** buffers, size_t* buffer_sizes, size_t* skip_counts) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Read if(tiledb_array->array_->read(buffers, buffer_sizes, skip_counts) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_evaluate_cell( const TileDB_Array* tiledb_array, void** buffers, size_t* buffer_sizes, int64_t* positions) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Evaluate cell int rc; if((rc = tiledb_array->array_->evaluate_cell(buffers, buffer_sizes, positions)) == TILEDB_AR_ERR) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // true/false depending on the evaluation return rc; } int tiledb_array_overflow( const TileDB_Array* tiledb_array, int attribute_id) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Check overflow return (int) tiledb_array->array_->overflow(attribute_id); } int tiledb_array_consolidate( const TileDB_CTX* tiledb_ctx, const char* array, size_t buffer_size, int batch_size) { // Check array name length if(array == NULL || strlen(array) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid array name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Consolidate if(tiledb_ctx->storage_manager_->array_consolidate(array, buffer_size, batch_size) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } else return TILEDB_OK; } int tiledb_array_finalize(TileDB_Array* tiledb_array) { // Sanity check if(!sanity_check(tiledb_array) || !sanity_check(tiledb_array->tiledb_ctx_)) return TILEDB_ERR; // Finalize array int rc = tiledb_array->tiledb_ctx_->storage_manager_->array_finalize( tiledb_array->array_); free(tiledb_array); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_sync(TileDB_Array* tiledb_array) { // Sanity check if(!sanity_check(tiledb_array) || !sanity_check(tiledb_array->tiledb_ctx_)) return TILEDB_ERR; // Sync int rc = tiledb_array->tiledb_ctx_->storage_manager_->array_sync( tiledb_array->array_); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_sync_attribute( TileDB_Array* tiledb_array, const char* attribute) { // Sanity check if(!sanity_check(tiledb_array) || !sanity_check(tiledb_array->tiledb_ctx_)) return TILEDB_ERR; // Sync attribute int rc = tiledb_array->tiledb_ctx_->storage_manager_->array_sync_attribute( tiledb_array->array_, attribute); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } typedef struct TileDB_ArrayIterator { ArrayIterator* array_it_; const TileDB_CTX* tiledb_ctx_; } TileDB_ArrayIterator; int tiledb_array_iterator_init( const TileDB_CTX* tiledb_ctx, TileDB_ArrayIterator** tiledb_array_it, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes) { return tiledb_array_iterator_init_with_filter( tiledb_ctx, tiledb_array_it, array, mode, subarray, attributes, attribute_num, buffers, buffer_sizes, ""); } int tiledb_array_iterator_init_with_filter( const TileDB_CTX* tiledb_ctx, TileDB_ArrayIterator** tiledb_array_it, const char* array, int mode, const void* subarray, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes, const char *filter_expression) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Allocate memory for the array iterator struct *tiledb_array_it = (TileDB_ArrayIterator*) malloc(sizeof(struct TileDB_ArrayIterator)); // Set TileDB context (*tiledb_array_it)->tiledb_ctx_ = tiledb_ctx; // Initialize the array iterator int rc = tiledb_ctx->storage_manager_->array_iterator_init( (*tiledb_array_it)->array_it_, array, mode, subarray, attributes, attribute_num, buffers, buffer_sizes, filter_expression); // Error if(rc != TILEDB_SM_OK) { free(*tiledb_array_it); strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_iterator_reset_subarray( TileDB_ArrayIterator* tiledb_array_it, const void* subarray) { int rc = tiledb_array_it->array_it_->reset_subarray(subarray); // Error if(rc != TILEDB_AIT_OK) { strcpy(tiledb_errmsg, tiledb_ait_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_iterator_get_value( TileDB_ArrayIterator* tiledb_array_it, int attribute_id, const void** value, size_t* value_size) { // Sanity check if(!sanity_check(tiledb_array_it)) return TILEDB_ERR; // Get value if(tiledb_array_it->array_it_->get_value( attribute_id, value, value_size) != TILEDB_AIT_OK) { strcpy(tiledb_errmsg, tiledb_ait_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_iterator_next( TileDB_ArrayIterator* tiledb_array_it) { // Sanity check if(!sanity_check(tiledb_array_it)) return TILEDB_ERR; // Advance iterator if(tiledb_array_it->array_it_->next() != TILEDB_AIT_OK) { strcpy(tiledb_errmsg, tiledb_ait_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_iterator_end( TileDB_ArrayIterator* tiledb_array_it) { // Sanity check if(!sanity_check(tiledb_array_it)) return TILEDB_ERR; // Check if the iterator reached the end return (int) tiledb_array_it->array_it_->end(); } int tiledb_array_iterator_finalize( TileDB_ArrayIterator* tiledb_array_it) { // Sanity check if(!sanity_check(tiledb_array_it)) return TILEDB_ERR; // Finalize array int rc = tiledb_array_it->tiledb_ctx_-> storage_manager_->array_iterator_finalize( tiledb_array_it->array_it_); free(tiledb_array_it); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_OK; } // Success return TILEDB_OK; } /* ****************************** */ /* METADATA */ /* ****************************** */ typedef struct TileDB_Metadata { Metadata* metadata_; const TileDB_CTX* tiledb_ctx_; } TileDB_Metadata; int tiledb_metadata_set_schema( TileDB_MetadataSchema* tiledb_metadata_schema, const char* metadata_name, const char** attributes, int attribute_num, int64_t capacity, const int* cell_val_num, const int* compression, const int* compression_level, const int* types) { // Sanity check if(tiledb_metadata_schema == NULL) { std::string errmsg = "Invalid metadata schema pointer"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } memset(tiledb_metadata_schema, 0, sizeof(TileDB_MetadataSchema)); // Set metadata name size_t metadata_name_len = strlen(metadata_name); if(metadata_name == NULL || metadata_name_len > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid metadata name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } tiledb_metadata_schema->metadata_name_ = (char*) malloc(metadata_name_len+1); strcpy(tiledb_metadata_schema->metadata_name_, metadata_name); /* Set attributes and number of attributes. */ tiledb_metadata_schema->attribute_num_ = attribute_num; tiledb_metadata_schema->attributes_ = (char**) malloc(attribute_num*sizeof(char*)); for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } tiledb_metadata_schema->attributes_[i] = (char*) malloc(attribute_len+1); strcpy(tiledb_metadata_schema->attributes_[i], attributes[i]); } // Set types tiledb_metadata_schema->types_ = (int*) malloc((attribute_num+1)*sizeof(int)); for(int i=0; itypes_[i] = types[i]; // Set cell val num if(cell_val_num == NULL) { tiledb_metadata_schema->cell_val_num_ = NULL; } else { tiledb_metadata_schema->cell_val_num_ = (int*) malloc((attribute_num)*sizeof(int)); for(int i=0; icell_val_num_[i] = cell_val_num[i]; } } // Set capacity tiledb_metadata_schema->capacity_ = capacity; // Set compression if(compression == NULL) { tiledb_metadata_schema->compression_ = NULL; } else { tiledb_metadata_schema->compression_ = (int*) malloc((attribute_num+1)*sizeof(int)); for(int i=0; icompression_[i] = compression[i]; } // Set compression level if(compression_level == NULL) { tiledb_metadata_schema->compression_level_ = NULL; } else { tiledb_metadata_schema->compression_level_ = (int*) malloc((attribute_num+1)*sizeof(int)); for(int i=0; icompression_level_[i] = compression_level[i]; } // Return return TILEDB_OK; } int tiledb_metadata_create( const TileDB_CTX* tiledb_ctx, const TileDB_MetadataSchema* metadata_schema) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Copy metadata schema to the proper struct MetadataSchemaC metadata_schema_c; memset(&metadata_schema_c, 0, sizeof(metadata_schema_c)); metadata_schema_c.metadata_name_ = metadata_schema->metadata_name_; metadata_schema_c.attributes_ = metadata_schema->attributes_; metadata_schema_c.attribute_num_ = metadata_schema->attribute_num_; metadata_schema_c.capacity_ = metadata_schema->capacity_; metadata_schema_c.cell_val_num_ = metadata_schema->cell_val_num_; metadata_schema_c.compression_ = metadata_schema->compression_; metadata_schema_c.compression_level_ = metadata_schema->compression_level_; metadata_schema_c.types_ = metadata_schema->types_; // Create the metadata if(tiledb_ctx->storage_manager_->metadata_create(&metadata_schema_c) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_init( const TileDB_CTX* tiledb_ctx, TileDB_Metadata** tiledb_metadata, const char* metadata, int mode, const char** attributes, int attribute_num) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Allocate memory for the array struct *tiledb_metadata = (TileDB_Metadata*) malloc(sizeof(struct TileDB_Metadata)); // Set TileDB context (*tiledb_metadata)->tiledb_ctx_ = tiledb_ctx; // Init the metadata if(tiledb_ctx->storage_manager_->metadata_init( (*tiledb_metadata)->metadata_, metadata, mode, attributes, attribute_num) != TILEDB_SM_OK) { free(*tiledb_metadata); strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_reset_attributes( const TileDB_Metadata* tiledb_metadata, const char** attributes, int attribute_num) { // Sanity check if(!sanity_check(tiledb_metadata)) return TILEDB_ERR; // Reset attributes if(tiledb_metadata->metadata_->reset_attributes( attributes, attribute_num) != TILEDB_MT_OK) { strcpy(tiledb_errmsg, tiledb_mt_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_get_schema( const TileDB_Metadata* tiledb_metadata, TileDB_MetadataSchema* tiledb_metadata_schema) { // Sanity check if(!sanity_check(tiledb_metadata)) return TILEDB_ERR; // Get the metadata schema MetadataSchemaC metadata_schema_c; tiledb_metadata->metadata_->array_schema()->array_schema_export( &metadata_schema_c); // Copy the metadata schema C struct to the output tiledb_metadata_schema->metadata_name_ = metadata_schema_c.metadata_name_; tiledb_metadata_schema->attributes_ = metadata_schema_c.attributes_; tiledb_metadata_schema->attribute_num_ = metadata_schema_c.attribute_num_; tiledb_metadata_schema->capacity_ = metadata_schema_c.capacity_; tiledb_metadata_schema->cell_val_num_ = metadata_schema_c.cell_val_num_; tiledb_metadata_schema->compression_ = metadata_schema_c.compression_; tiledb_metadata_schema->compression_level_ = metadata_schema_c.compression_level_; tiledb_metadata_schema->types_ = metadata_schema_c.types_; // Success return TILEDB_OK; } int tiledb_metadata_load_schema( const TileDB_CTX* tiledb_ctx, const char* metadata, TileDB_MetadataSchema* tiledb_metadata_schema) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check metadata name length if(metadata == NULL || strlen(metadata) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid metadata name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Get the array schema ArraySchema* array_schema; if(tiledb_ctx->storage_manager_->metadata_load_schema( metadata, array_schema) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } MetadataSchemaC metadata_schema_c; array_schema->array_schema_export(&metadata_schema_c); // Copy the metadata schema C struct to the output tiledb_metadata_schema->metadata_name_ = metadata_schema_c.metadata_name_; tiledb_metadata_schema->attributes_ = metadata_schema_c.attributes_; tiledb_metadata_schema->attribute_num_ = metadata_schema_c.attribute_num_; tiledb_metadata_schema->capacity_ = metadata_schema_c.capacity_; tiledb_metadata_schema->cell_val_num_ = metadata_schema_c.cell_val_num_; tiledb_metadata_schema->compression_ = metadata_schema_c.compression_; tiledb_metadata_schema->compression_level_ = metadata_schema_c.compression_level_; tiledb_metadata_schema->types_ = metadata_schema_c.types_; // Clean up delete array_schema; // Success return TILEDB_OK; } int tiledb_metadata_free_schema( TileDB_MetadataSchema* tiledb_metadata_schema) { // Trivial case if(tiledb_metadata_schema == NULL) return TILEDB_OK; // Free name if(tiledb_metadata_schema->metadata_name_ != NULL) free(tiledb_metadata_schema->metadata_name_); // Free attributes if(tiledb_metadata_schema->attributes_ != NULL) { for(int i=0; iattribute_num_; ++i) if(tiledb_metadata_schema->attributes_[i] != NULL) free(tiledb_metadata_schema->attributes_[i]); free(tiledb_metadata_schema->attributes_); } // Free types if(tiledb_metadata_schema->types_ != NULL) free(tiledb_metadata_schema->types_); // Free compression if(tiledb_metadata_schema->compression_ != NULL) free(tiledb_metadata_schema->compression_); // Free compression level if(tiledb_metadata_schema->compression_level_ != NULL) free(tiledb_metadata_schema->compression_level_); // Free cell val num if(tiledb_metadata_schema->cell_val_num_ != NULL) free(tiledb_metadata_schema->cell_val_num_); // Success return TILEDB_OK; } int tiledb_metadata_write( const TileDB_Metadata* tiledb_metadata, const char* keys, size_t keys_size, const void** buffers, const size_t* buffer_sizes) { // Sanity check if(!sanity_check(tiledb_metadata)) return TILEDB_ERR; // Write if(tiledb_metadata->metadata_->write( keys, keys_size, buffers, buffer_sizes) != TILEDB_MT_OK) { strcpy(tiledb_errmsg, tiledb_mt_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_read( const TileDB_Metadata* tiledb_metadata, const char* key, void** buffers, size_t* buffer_sizes) { // Sanity check if(!sanity_check(tiledb_metadata)) return TILEDB_ERR; // Read if(tiledb_metadata->metadata_->read( key, buffers, buffer_sizes) != TILEDB_MT_OK) { strcpy(tiledb_errmsg, tiledb_mt_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_overflow( const TileDB_Metadata* tiledb_metadata, int attribute_id) { // Sanity check if(!sanity_check(tiledb_metadata)) return TILEDB_ERR; return (int) tiledb_metadata->metadata_->overflow(attribute_id); } int tiledb_metadata_consolidate( const TileDB_CTX* tiledb_ctx, const char* metadata) { // Check metadata name length if(metadata == NULL || strlen(metadata) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid metadata name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Consolidate if(tiledb_ctx->storage_manager_->metadata_consolidate(metadata) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_finalize(TileDB_Metadata* tiledb_metadata) { // Sanity check if(!sanity_check(tiledb_metadata)) return TILEDB_ERR; // Finalize metadata int rc = tiledb_metadata->tiledb_ctx_->storage_manager_->metadata_finalize( tiledb_metadata->metadata_); // Clean up free(tiledb_metadata); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } typedef struct TileDB_MetadataIterator { MetadataIterator* metadata_it_; const TileDB_CTX* tiledb_ctx_; } TileDB_MetadataIterator; int tiledb_metadata_iterator_init( const TileDB_CTX* tiledb_ctx, TileDB_MetadataIterator** tiledb_metadata_it, const char* metadata, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Allocate memory for the metadata struct *tiledb_metadata_it = (TileDB_MetadataIterator*) malloc(sizeof(struct TileDB_MetadataIterator)); // Set TileDB context (*tiledb_metadata_it)->tiledb_ctx_ = tiledb_ctx; // Initialize the metadata iterator if(tiledb_ctx->storage_manager_->metadata_iterator_init( (*tiledb_metadata_it)->metadata_it_, metadata, attributes, attribute_num, buffers, buffer_sizes) != TILEDB_SM_OK) { free(*tiledb_metadata_it); strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_iterator_get_value( TileDB_MetadataIterator* tiledb_metadata_it, int attribute_id, const void** value, size_t* value_size) { // Sanity check if(!sanity_check(tiledb_metadata_it)) return TILEDB_ERR; // Get value if(tiledb_metadata_it->metadata_it_->get_value( attribute_id, value, value_size) != TILEDB_MIT_OK) { strcpy(tiledb_errmsg, tiledb_mit_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_iterator_next( TileDB_MetadataIterator* tiledb_metadata_it) { // Sanity check if(!sanity_check(tiledb_metadata_it)) return TILEDB_ERR; // Advance metadata iterator if(tiledb_metadata_it->metadata_it_->next() != TILEDB_MIT_OK) { strcpy(tiledb_errmsg, tiledb_mit_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_metadata_iterator_end( TileDB_MetadataIterator* tiledb_metadata_it) { // Sanity check if(!sanity_check(tiledb_metadata_it)) return TILEDB_ERR; // Check if the metadata iterator reached its end return (int) tiledb_metadata_it->metadata_it_->end(); } int tiledb_metadata_iterator_finalize( TileDB_MetadataIterator* tiledb_metadata_it) { // Sanity check if(!sanity_check(tiledb_metadata_it)) return TILEDB_ERR; // Finalize metadata iterator int rc = tiledb_metadata_it->tiledb_ctx_-> storage_manager_->metadata_iterator_finalize( tiledb_metadata_it->metadata_it_); // Clean up free(tiledb_metadata_it); // Error if(rc != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } /* ****************************** */ /* DIRECTORY MANAGEMENT */ /* ****************************** */ int tiledb_clear( const TileDB_CTX* tiledb_ctx, const char* dir) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check directory name length if(dir == NULL || strlen(dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid directory name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Clear if(tiledb_ctx->storage_manager_->clear(dir) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_delete( const TileDB_CTX* tiledb_ctx, const char* dir) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check directory name length if(dir == NULL || strlen(dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid directory name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Delete if(tiledb_ctx->storage_manager_->delete_entire(dir) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_move( const TileDB_CTX* tiledb_ctx, const char* old_dir, const char* new_dir) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check old directory name length if(old_dir == NULL || strlen(old_dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid old directory name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Check new directory name length if(new_dir == NULL || strlen(new_dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid new directory name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // Move if(tiledb_ctx->storage_manager_->move(old_dir, new_dir) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_ls_workspaces( const TileDB_CTX* tiledb_ctx, const char* parent_dir, char** workspaces, int* workspace_num) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // List workspaces if(tiledb_ctx->storage_manager_->ls_workspaces( parent_dir, workspaces, *workspace_num) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_ls_workspaces_c( const TileDB_CTX* tiledb_ctx, const char* parent_dir, int* workspace_num) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // List workspaces if(tiledb_ctx->storage_manager_->ls_workspaces_c( parent_dir, *workspace_num) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_ls( const TileDB_CTX* tiledb_ctx, const char* parent_dir, char** dirs, int* dir_types, int* dir_num) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check parent directory name length if(parent_dir == NULL || strlen(parent_dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid parent directory name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // List TileDB objects if(tiledb_ctx->storage_manager_->ls( parent_dir, dirs, dir_types, *dir_num) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_ls_c( const TileDB_CTX* tiledb_ctx, const char* parent_dir, int* dir_num) { // Sanity check if(!sanity_check(tiledb_ctx)) return TILEDB_ERR; // Check parent directory name length if(parent_dir == NULL || strlen(parent_dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid parent directory name length"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return TILEDB_ERR; } // List TileDB objects if(tiledb_ctx->storage_manager_->ls_c( parent_dir, *dir_num) != TILEDB_SM_OK) { strcpy(tiledb_errmsg, tiledb_sm_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } /* ****************************** */ /* ASYNCHRONOUS I/O (AIO */ /* ****************************** */ int tiledb_array_aio_read( const TileDB_Array* tiledb_array, TileDB_AIO_Request* tiledb_aio_request) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Copy the AIO request AIO_Request* aio_request = (AIO_Request*) malloc(sizeof(struct AIO_Request)); aio_request->id_ = (size_t) tiledb_aio_request; aio_request->buffers_ = tiledb_aio_request->buffers_; aio_request->buffer_sizes_ = tiledb_aio_request->buffer_sizes_; aio_request->mode_ = tiledb_array->array_->mode(); aio_request->status_ = &(tiledb_aio_request->status_); aio_request->subarray_ = tiledb_aio_request->subarray_; aio_request->completion_handle_ = tiledb_aio_request->completion_handle_; aio_request->completion_data_ = tiledb_aio_request->completion_data_; // Submit the AIO read request if(tiledb_array->array_->aio_read(aio_request) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } int tiledb_array_aio_write( const TileDB_Array* tiledb_array, TileDB_AIO_Request* tiledb_aio_request) { // Sanity check if(!sanity_check(tiledb_array)) return TILEDB_ERR; // Copy the AIO request AIO_Request* aio_request = (AIO_Request*) malloc(sizeof(struct AIO_Request)); aio_request->id_ = (size_t) tiledb_aio_request; aio_request->buffers_ = tiledb_aio_request->buffers_; aio_request->buffer_sizes_ = tiledb_aio_request->buffer_sizes_; aio_request->mode_ = tiledb_array->array_->mode(); aio_request->status_ = &(tiledb_aio_request->status_); aio_request->subarray_ = tiledb_aio_request->subarray_; aio_request->completion_handle_ = tiledb_aio_request->completion_handle_; aio_request->completion_data_ = tiledb_aio_request->completion_data_; // Submit the AIO write request if(tiledb_array->array_->aio_write(aio_request) != TILEDB_AR_OK) { strcpy(tiledb_errmsg, tiledb_ar_errmsg.c_str()); return TILEDB_ERR; } // Success return TILEDB_OK; } /* *********************************************************************** */ /* Expose some filesystem functionality implemented in TileDB */ /* *********************************************************************** */ inline bool sanity_check_fs(const TileDB_CTX* tiledb_ctx) { if (tiledb_ctx && tiledb_ctx->storage_manager_ && tiledb_ctx->storage_manager_->get_config() && tiledb_ctx->storage_manager_->get_config()->get_filesystem()) { return true; } std::string errmsg = "TileDB configured incorrectly"; PRINT_ERROR(errmsg); strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } inline bool invoke_bool_fs_fn(const TileDB_CTX* tiledb_ctx, const std::string& dir, bool (*fn)(StorageFS*, const std::string&)) { if (sanity_check_fs(tiledb_ctx)) { tiledb_fs_errmsg.clear(); bool rc = fn(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), dir); if (!tiledb_fs_errmsg.empty()) strcpy(tiledb_errmsg, tiledb_fs_errmsg.c_str()); return rc; } std::string errmsg = "Could not invoke TileDB functionality. Check TileDB configuration"; strcpy(tiledb_errmsg, (TILEDB_ERRMSG + errmsg).c_str()); return false; } bool is_workspace(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_bool_fs_fn(tiledb_ctx, dir, &is_workspace); } bool is_group(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_bool_fs_fn(tiledb_ctx, dir, &is_group); } bool is_array(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_bool_fs_fn(tiledb_ctx, dir, &is_array); } bool is_fragment(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_bool_fs_fn(tiledb_ctx, dir, &is_fragment); } bool is_metadata(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_bool_fs_fn(tiledb_ctx, dir, &is_metadata); } bool is_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_bool_fs_fn(tiledb_ctx, dir, &is_dir); } std::string real_dir(const TileDB_CTX* tiledb_ctx, const std::string& dirpath) { if (sanity_check_fs(tiledb_ctx)) { return real_dir(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), dirpath); } return dirpath; } bool is_file(const TileDB_CTX* tiledb_ctx, const std::string& file) { return invoke_bool_fs_fn(tiledb_ctx, file, &is_file); } std::string parent_dir(const std::string& path) { return parent_dir(NULL, path); } std::string current_working_dir(const TileDB_CTX* tiledb_ctx) { if (sanity_check_fs(tiledb_ctx)) { return current_dir(tiledb_ctx->storage_manager_->get_config()->get_filesystem()); } return ""; } ssize_t file_size(const TileDB_CTX* tiledb_ctx, const std::string& file) { if (sanity_check_fs(tiledb_ctx)) { return file_size(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), file); } return TILEDB_ERR; } inline int invoke_int_fs_fn(const TileDB_CTX* tiledb_ctx, const std::string& dir, int (*fn)(StorageFS*, const std::string&)) { if (sanity_check_fs(tiledb_ctx)) { tiledb_fs_errmsg.clear(); int rc = fn(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), dir); if (!tiledb_fs_errmsg.empty()) strcpy(tiledb_errmsg, tiledb_fs_errmsg.c_str()); return rc; } return TILEDB_ERR; } int set_working_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_int_fs_fn(tiledb_ctx, dir, &set_working_dir); } int create_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_int_fs_fn(tiledb_ctx, dir, &create_dir); } int delete_dir(const TileDB_CTX* tiledb_ctx, const std::string& dir) { return invoke_int_fs_fn(tiledb_ctx, dir, &delete_dir); } std::vector get_dirs(const TileDB_CTX* tiledb_ctx, const std::string& dir) { if (sanity_check_fs(tiledb_ctx)) { return get_dirs(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), dir); } return std::vector{}; } std::vector get_files(const TileDB_CTX* tiledb_ctx, const std::string& dir) { if (sanity_check_fs(tiledb_ctx)) { return get_files(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), dir); } return std::vector{}; } int create_file(const TileDB_CTX* tiledb_ctx, const std::string& filename, int flags, mode_t mode) { if (sanity_check_fs(tiledb_ctx)) { return create_file(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), filename, flags, mode); } return TILEDB_ERR; } int read_file(const TileDB_CTX* tiledb_ctx, const std::string& filename, off_t offset, void *buffer, size_t length) { if (sanity_check_fs(tiledb_ctx)) { if (!read_from_file(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), filename, offset, buffer, length)) { return TILEDB_OK; } strcpy(tiledb_errmsg, tiledb_fs_errmsg.c_str()); } return TILEDB_ERR; } int write_file(const TileDB_CTX* tiledb_ctx, const std::string& filename, const void *buffer, size_t buffer_size) { if (sanity_check(tiledb_ctx)) { if (!write_to_file(tiledb_ctx->storage_manager_->get_config()->get_filesystem(), filename, buffer, buffer_size)) { return TILEDB_OK; } strcpy(tiledb_errmsg, tiledb_fs_errmsg.c_str()); } return TILEDB_ERR; } int delete_file(const TileDB_CTX* tiledb_ctx, const std::string& filename) { return invoke_int_fs_fn(tiledb_ctx, filename, &delete_file); } int close_file(const TileDB_CTX* tiledb_ctx, const std::string& filename) { return invoke_int_fs_fn(tiledb_ctx, filename, &close_file); } genomicsdb-0.0~git20231212.9d7ddd0/core/src/c_api/tiledb_utils.cc000066400000000000000000000360511453617025200241360ustar00rootroot00000000000000/** * @file tiledb_utils.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 Omics Data Automation Inc. and Intel Corporation * @copyright Copyright (c) 2019-2021 Omics Data Automation Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Utility functions to access TileDB functionality in an opaque fashion. These * convenience functions do not require TileDB_CTX handle. * */ #include "tiledb_utils.h" #include "codec.h" #include "tiledb_storage.h" #include "storage_fs.h" #include #include #include #include #include #include namespace TileDBUtils { static int setup(TileDB_CTX **ptiledb_ctx, const std::string& home, const bool enable_shared_posixfs_optimizations=false) { int rc; TileDB_Config tiledb_config = {}; tiledb_config.home_ = strdup(home.c_str()); tiledb_config.enable_shared_posixfs_optimizations_ = enable_shared_posixfs_optimizations; rc = tiledb_ctx_init(ptiledb_ctx, &tiledb_config); free(const_cast(tiledb_config.home_)); return rc; } static int finalize(TileDB_CTX *tiledb_ctx) { return tiledb_ctx_finalize(tiledb_ctx); } bool is_cloud_path(const std::string& path) { return path.find("://") != std::string::npos; } std::string get_path(const std::string &path) { std::size_t check_cloud = path.find("://"); if (check_cloud != std::string::npos && path.substr(0, check_cloud).compare("hdfs") != 0) return uri(path).path(); return path; } std::string append_path(const std::string& dir, const std::string& path) { std::size_t query_pos = dir.find('?'); if (query_pos == std::string::npos) { return StorageFS::slashify(dir) + path; } else { return StorageFS::slashify(dir.substr(0, query_pos)) + path + dir.substr(query_pos); } } /** * Returns 0 when workspace is created * -1 when path is not a directory * -2 when workspace could not be created * 1 when workspace exists and nothing is changed */ #define OK 0 #define NOT_DIR -1 #define NOT_CREATED -2 #define UNCHANGED 1 int initialize_workspace(TileDB_CTX **ptiledb_ctx, const std::string& workspace, const bool replace, const bool enable_shared_posixfs_optimizations) { *ptiledb_ctx = NULL; int rc; rc = setup(ptiledb_ctx, workspace, enable_shared_posixfs_optimizations); std::string workspace_path = get_path(workspace); if (rc) { return NOT_CREATED; } if (is_file(*ptiledb_ctx, workspace_path)) { return NOT_DIR; } if (is_workspace(*ptiledb_ctx, workspace_path)) { if (replace) { if (is_dir(*ptiledb_ctx, workspace_path) && delete_dir(*ptiledb_ctx, workspace_path) ) { return NOT_CREATED; } } else { return UNCHANGED; } } rc = tiledb_workspace_create(*ptiledb_ctx, workspace_path.c_str()); if (rc != TILEDB_OK) { rc = NOT_CREATED; } else { rc = OK; } return rc; } #define FINALIZE \ do { \ if (tiledb_ctx) { \ finalize(tiledb_ctx); \ } \ } while(false) int create_workspace(const std::string& workspace, bool replace) { TileDB_CTX *tiledb_ctx; int rc = initialize_workspace(&tiledb_ctx, workspace, replace); FINALIZE; return rc; } bool workspace_exists(const std::string& workspace) { bool exists = false; TileDB_CTX *tiledb_ctx; int rc = setup(&tiledb_ctx, workspace); exists = !rc && is_workspace(tiledb_ctx, get_path(workspace)); FINALIZE; return exists; } bool array_exists(const std::string& workspace, const std::string& array_name) { bool exists = false; TileDB_CTX *tiledb_ctx; int rc = setup(&tiledb_ctx, workspace); exists = !rc && is_array(tiledb_ctx, StorageFS::append_paths(get_path(workspace), array_name)); FINALIZE; return exists; } std::vector get_array_names(const std::string& workspace) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, workspace)) { FINALIZE; return std::vector{}; } std::vector array_names; std::vector dirs = get_dirs(tiledb_ctx, workspace); if (!dirs.empty()) { for (std::vector::iterator dir = dirs.begin() ; dir != dirs.end(); dir++) { std::string path(*dir); if (is_array(tiledb_ctx, path)) { size_t pos = path.find_last_of("\\/"); if (pos == std::string::npos) { array_names.push_back(path); } else { array_names.push_back(path.substr(pos+1)); } } } } finalize(tiledb_ctx); return array_names; } std::vector get_fragment_names(const std::string& workspace) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, workspace)) { FINALIZE; return std::vector{}; } std::vector fragment_names; std::vector dirs = get_dirs(tiledb_ctx, workspace); if (!dirs.empty()) { for (std::vector::iterator dir = dirs.begin() ; dir != dirs.end(); dir++) { std::string path(*dir); if (is_array(tiledb_ctx, path)) { std::vector fragment_dirs = get_dirs(tiledb_ctx, path); if (!fragment_dirs.empty()) { for (std::vector::iterator fragment_dir = fragment_dirs.begin(); fragment_dir != fragment_dirs.end(); fragment_dir++) { std::string fragment_path(*fragment_dir); if (is_fragment(tiledb_ctx, fragment_path)) { size_t pos = fragment_path.find_last_of("\\/"); if (pos == std::string::npos) { fragment_names.push_back(fragment_path); } else { fragment_names.push_back(fragment_path.substr(pos+1)); } } } } } } } finalize(tiledb_ctx); return fragment_names; } bool is_dir(const std::string& dirpath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(dirpath))) { FINALIZE; return false; } bool check = is_dir(tiledb_ctx, dirpath); finalize(tiledb_ctx); return check; } std::string real_dir(const std::string& dirpath) { if (is_cloud_path(dirpath)) { return dirpath; } else { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(dirpath))) { FINALIZE; return dirpath; } std::string real_dirpath = real_dir(tiledb_ctx, dirpath); finalize(tiledb_ctx); return real_dirpath; } } int create_dir(const std::string& dirpath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(dirpath))) { FINALIZE; return TILEDB_ERR; } int rc = create_dir(tiledb_ctx, dirpath); finalize(tiledb_ctx); return rc; } int delete_dir(const std::string& dirpath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(dirpath))) { FINALIZE; return TILEDB_ERR; } int rc = delete_dir(tiledb_ctx, dirpath); finalize(tiledb_ctx); return rc; } bool is_file(const std::string& filepath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(filepath))) { FINALIZE; return false; } bool check = is_file(tiledb_ctx, filepath); finalize(tiledb_ctx); return check; } ssize_t file_size(const std::string& filepath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(filepath))) { FINALIZE; return false; } ssize_t filesize = file_size(tiledb_ctx, filepath); finalize(tiledb_ctx); return filesize; } std::vector get_dirs(const std::string& dirpath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(dirpath))) { FINALIZE; return {}; } auto dirs = get_dirs(tiledb_ctx, dirpath); finalize(tiledb_ctx); return dirs; } std::vector get_files(const std::string& dirpath) { TileDB_CTX *tiledb_ctx; if (setup(&tiledb_ctx, parent_dir(dirpath))) { FINALIZE; return {}; } auto files = get_files(tiledb_ctx, dirpath);; finalize(tiledb_ctx); return files; } static int check_file(TileDB_CTX *tiledb_ctx, std::string filename) { if (is_dir(tiledb_ctx, filename)) { snprintf(tiledb_errmsg, TILEDB_ERRMSG_MAX_LEN, "File path=%s exists as a directory\n", filename.c_str()); return TILEDB_ERR; } return TILEDB_OK; } static int check_file_for_read(TileDB_CTX *tiledb_ctx, std::string filename) { if (check_file(tiledb_ctx, filename)) { return TILEDB_ERR; } if (!is_file(tiledb_ctx, filename) || file_size(tiledb_ctx, filename) == 0) { snprintf(tiledb_errmsg, TILEDB_ERRMSG_MAX_LEN, "File path=%s does not exist or is empty\n", filename.c_str()); return TILEDB_ERR; } return TILEDB_OK; } #include "tiledb_openssl_shim.h" void print_md5_hash(unsigned char* buffer, size_t length) { unsigned char md[MD5_DIGEST_LENGTH]; if(OpenSSL_version_num() < 0x30000000L) { MD5(buffer, length, md); } else { EVP_MD_CTX* mdctx; mdctx = EVP_MD_CTX_new(); EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); EVP_DigestUpdate(mdctx, buffer, length); EVP_DigestFinal_ex(mdctx, md, NULL); EVP_MD_CTX_free(mdctx); } for(auto i=0; i (handle))->do_compress_tile(segment, segment_size, compressed_segment, compressed_segment_size); } int decompress(void *handle, unsigned char* compressed_segment, size_t compressed_segment_size, unsigned char* segment, size_t segment_size) { return (reinterpret_cast(handle))->do_decompress_tile(compressed_segment, compressed_segment_size, segment, segment_size); } void finalize_codec(void *handle) { delete reinterpret_cast(handle); } } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/000077500000000000000000000000001453617025200211415ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec.cc000066400000000000000000000243441453617025200225340ustar00rootroot00000000000000/** * @file codec.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the base class for compression and decompression. */ #include "codec.h" #include "codec_gzip.h" #ifdef ENABLE_ZSTD # include "codec_zstd.h" #endif #include "codec_lz4.h" #ifdef ENABLE_BLOSC # include "codec_blosc.h" #endif #include "codec_rle.h" #include "codec_filter.h" #include "codec_filter_bit_shuffle.h" #include "codec_filter_delta_encode.h" #include "tiledb.h" /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_CD_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_cd_errmsg = ""; /* ****************************** */ /* FACTORY METHODS */ /* ****************************** */ #ifdef ENABLE_BLOSC static std::string get_blosc_compressor(const int compression_type) { switch (compression_type) { case TILEDB_BLOSC: return "blosclz"; case TILEDB_BLOSC_LZ4: return "lz4"; case TILEDB_BLOSC_LZ4HC: return "lz4hc"; case TILEDB_BLOSC_SNAPPY: return "snappy"; case TILEDB_BLOSC_ZLIB: return "zlib"; case TILEDB_BLOSC_ZSTD: return "zstd"; default: return ""; }; } #endif int get_filter_type(const ArraySchema* array_schema, const int attribute_id, const bool is_offsets_compression, filter_type_t filter_type) { if (is_offsets_compression) { return array_schema->offsets_compression(attribute_id) & filter_type; } else { return array_schema->compression(attribute_id) & filter_type; } } int get_filter_level(const ArraySchema* array_schema, const int attribute_id, const bool is_offsets_compression) { if (is_offsets_compression) { return array_schema->offsets_compression_level(attribute_id); } else { return array_schema->compression_level(attribute_id); } } static std::mutex registered_codecs_mutex_; static std::map registered_codecs_; int Codec::register_codec(int compression_type, Codec::create_fn_t create_fn) { const std::lock_guard lock(registered_codecs_mutex_); if (is_registered_codec(compression_type)) { PRINT_ERROR("Codec for compression type=" + std::to_string(compression_type) + " has already been registered"); return TILEDB_CD_ERR; } registered_codecs_.insert({compression_type, create_fn}); return TILEDB_CD_OK; } bool Codec::is_registered_codec(int compression_type) { return registered_codecs_.find(compression_type) != registered_codecs_.end(); } Codec::create_fn_t Codec::get_registered_codec(int compression_type) { const std::lock_guard lock(registered_codecs_mutex_); auto registered_codec = registered_codecs_.find(compression_type); if (registered_codec != registered_codecs_.end()) { return registered_codec->second; } return nullptr; } Codec* Codec::create(const ArraySchema* array_schema, const int attribute_id, const bool is_offsets_compression) { int compression_type = get_filter_type(array_schema, attribute_id, is_offsets_compression, COMPRESS); if (compression_type == TILEDB_NO_COMPRESSION) { return NULL; } // Check if there is an external creator registered first auto codec_creator = get_registered_codec(compression_type); if (codec_creator != nullptr) { return codec_creator(array_schema, attribute_id, is_offsets_compression); } int compression_level = get_filter_level(array_schema, attribute_id, is_offsets_compression); Codec* codec = NULL; switch (compression_type) { case TILEDB_GZIP: codec = new CodecGzip(compression_level); break; #ifdef ENABLE_ZSTD case TILEDB_ZSTD: codec = new CodecZStandard(compression_level); break; #endif case TILEDB_LZ4: codec = new CodecLZ4(compression_level); break; #ifdef ENABLE_BLOSC case TILEDB_BLOSC: case TILEDB_BLOSC_LZ4: case TILEDB_BLOSC_LZ4HC: case TILEDB_BLOSC_SNAPPY: case TILEDB_BLOSC_ZLIB: case TILEDB_BLOSC_ZSTD: { size_t type_size; if (is_offsets_compression) { type_size = sizeof(size_t); } else { type_size = array_schema->type_size(attribute_id); } codec = new CodecBlosc(compression_level, get_blosc_compressor(compression_type), type_size); break; } #endif case TILEDB_RLE: { int attribute_num = array_schema->attribute_num(); int dim_num = array_schema->dim_num(); int cell_order = array_schema->cell_order(); bool is_coords = (attribute_id == attribute_num); // TODO: visit offsets compression for RLE size_t value_size = (array_schema->var_size(attribute_id) || is_coords) ? array_schema->type_size(attribute_id) : array_schema->cell_size(attribute_id); codec = new CodecRLE(attribute_num, dim_num, cell_order, is_coords, value_size); break; } default: // TODO throw exception std::cerr << "Unsupported compression type:" << compression_type << "\n"; return NULL; } int pre_compress_type = get_filter_type(array_schema, attribute_id, is_offsets_compression, PRE_COMPRESS); switch (pre_compress_type) { case 0: break; case TILEDB_DELTA_ENCODE: CodecFilter* filter; if (array_schema->attribute(attribute_id) == TILEDB_COORDS) { filter = new CodecDeltaEncode(array_schema->type(attribute_id), array_schema->dim_num()); } else if (is_offsets_compression) { filter = new CodecDeltaEncode(TILEDB_UINT64, 1); } else { filter = new CodecDeltaEncode(array_schema->type(attribute_id), array_schema->cell_val_num(attribute_id)); } codec->set_pre_compression(filter); break; case TILEDB_BIT_SHUFFLE: codec->set_pre_compression(new CodecBitShuffle(array_schema->type(attribute_id))); break; default: std::cerr << "Unsupported pre-compression filter: " << pre_compress_type << "\n"; } int post_compress_type = get_filter_type(array_schema, attribute_id, is_offsets_compression, POST_COMPRESS); switch (post_compress_type) { case 0: break; default: std::cerr << "Unsupported post-compression filter: " << post_compress_type << "\n"; } return codec; } // Generalized non-TileDB codec usage int Codec::create(void **handle, int compression_type, int compression_level) { int rc = TILEDB_OK; switch (compression_type) { case TILEDB_GZIP: *handle = new CodecGzip(compression_level); break; #ifdef ENABLE_ZSTD case TILEDB_ZSTD: *handle = new CodecZStandard(compression_level); break; #endif default: snprintf(tiledb_errmsg, TILEDB_ERRMSG_MAX_LEN, "Compression algorithm %d not supported", compression_type); *handle = NULL; rc = TILEDB_ERR; } return rc; } int Codec::get_default_level(int compression_type) { switch(compression_type) { case TILEDB_GZIP: return TILEDB_COMPRESSION_LEVEL_GZIP; case TILEDB_ZSTD: return TILEDB_COMPRESSION_LEVEL_ZSTD; case TILEDB_BLOSC: return TILEDB_COMPRESSION_LEVEL_BLOSC; default: return -1; } } int Codec::print_errmsg(const std::string& msg) { if (msg.length() > 0) { PRINT_ERROR(msg); tiledb_cd_errmsg = TILEDB_CD_ERRMSG + msg; } return TILEDB_CD_ERR; } int Codec::compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { unsigned char* tile_precompressed = tile; if (pre_compression_filter_) { if (pre_compression_filter_->code(tile, tile_size) != 0) { return print_errmsg("Could not apply filter " + pre_compression_filter_->name() + " before compressing"); } if (!pre_compression_filter_->in_place()) { if (pre_compression_filter_->buffer() == NULL) { return print_errmsg("Error from precompression filter " + pre_compression_filter_->name()); } tile_precompressed = pre_compression_filter_->buffer(); } } if (do_compress_tile(tile_precompressed, tile_size, tile_compressed, tile_compressed_size)) { return print_errmsg("Could not compress with " + name()); } return TILEDB_CD_OK; } int Codec::decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { unsigned char* buffer = tile; if (pre_compression_filter_ && !pre_compression_filter_->in_place()) { if (pre_compression_filter_->allocate_buffer(tile_size) != TILEDB_CDF_OK) { return print_errmsg("OOM while trying to allocate memory for decompress using " + pre_compression_filter_->name()); } buffer = pre_compression_filter_->buffer(); } if (do_decompress_tile(tile_compressed, tile_compressed_size, buffer, tile_size)) { return print_errmsg("Could not decompress with " + name()); } if (pre_compression_filter_) { if (pre_compression_filter_->decode(tile, tile_size) != 0) { return print_errmsg("Could not apply filter " + pre_compression_filter_->name() + " after decompressing"); } } return TILEDB_CD_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_blosc.cc000066400000000000000000000064101453617025200237100ustar00rootroot00000000000000/** * @file codec_blosc.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements codec for Blosc for compression and decompression. */ #ifdef ENABLE_BLOSC #define BLOSC_EXTERN_DECL extern #include "codec_blosc.h" int CodecBlosc::do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { // Allocate space to store the compressed tile #define BLOSC_MAX_OVERHEAD 16 size_t compress_bound = tile_size + BLOSC_MAX_OVERHEAD; if(tile_compressed_ == NULL) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = malloc(compress_bound); } // Expand comnpressed tile if necessary if(compress_bound > tile_compressed_allocated_size_) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = realloc(tile_compressed_, compress_bound); } // Initialize Blosc blosc_init(); // Set the appropriate compressor if(blosc_set_compressor(compressor_.c_str()) < 0) { blosc_destroy(); return print_errmsg("Failed to set Blosc compressor"); } // Compress tile int blosc_size = blosc_compress( compression_level_, //clevel 1, //doshuffle type_size_, //typesize tile_size, //nbytes tile, //src static_cast(tile_compressed_), //dest tile_compressed_allocated_size_ //dest_size ); if(blosc_size < 0) { blosc_destroy(); return print_errmsg("Failed compressing with Blosc"); } *tile_compressed = tile_compressed_; tile_compressed_size = blosc_size; // Clean up blosc_destroy(); // Success return TILEDB_CD_OK; } int CodecBlosc::do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { // Initialization blosc_init(); // Decompress tile if(blosc_decompress( (const char*) tile_compressed, (char*) tile, tile_size) < 0) { blosc_destroy(); return print_errmsg("Blosc decompression failed"); } // Clean up blosc_destroy(); // Success return TILEDB_CD_OK; } #endif genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_filter.cc000066400000000000000000000037021453617025200240740ustar00rootroot00000000000000/** * @file codec_filter.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the base class for pre and post compression filters */ #include "codec_filter.h" #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_CDF_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_cdf_errmsg = ""; int CodecFilter::print_errmsg(const std::string& msg) { if (msg.length() > 0) { PRINT_ERROR(msg); tiledb_cdf_errmsg = TILEDB_CDF_ERRMSG + msg; } return TILEDB_CDF_ERR; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_filter_bit_shuffle.cc000066400000000000000000000100721453617025200264440ustar00rootroot00000000000000/** * @file codec_filter_bit_shuffle.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the Bit Shuffle Pre-Compression Filter. */ #include "bitshuffle_core.h" #include "codec_filter_bit_shuffle.h" #include "tiledb_constants.h" const std::string err_msg(int rc) { switch (rc) { case -1: return "Fail to allocate memory"; case -11: return "Missing SSE"; case -12: return "Missing AVX"; case -80: return "Input size not a multiple of 8"; case -81: return "Block Size not a multiple of 8"; case -91: return "Decompression error, wrong number of bytes processed"; default: return "Internal error"; } } template int do_code(T* tile, size_t tile_size_in_bytes, CodecFilter* filter) { if (tile_size_in_bytes%sizeof(T) != 0) { return filter->print_errmsg("Tile size to pre-compression filter " + filter->name() + " should be a multiple of sizeof type"); } if (filter->allocate_buffer(tile_size_in_bytes) == TILEDB_CDF_ERR) { return filter->print_errmsg("OOM while tring to allocate memory for filter " + filter->name()); } int rc = bshuf_bitshuffle(tile, filter->buffer(), tile_size_in_bytes/sizeof(T), sizeof(T), 0); if (rc < 0) { return filter->print_errmsg("Bit shuffle error: " + err_msg(rc)); } return TILEDB_CDF_OK; } int CodecBitShuffle::code(unsigned char* tile, size_t tile_size) { switch (type_) { case TILEDB_INT32: return do_code(reinterpret_cast(tile), tile_size, this); case TILEDB_INT64: return do_code(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT32: return do_code(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT64: return do_code(reinterpret_cast(tile), tile_size, this); default: return print_errmsg("CodecBitShuffle not implemented for type"); } } template int do_decode(T* tile, size_t tile_size_in_bytes, CodecFilter* filter) { if (tile_size_in_bytes%sizeof(T) != 0) { return filter->print_errmsg("Tile size to pre-compression filter " + filter->name() + " should be a multiple of sizeof type"); } int rc = bshuf_bitunshuffle(filter->buffer(), tile, tile_size_in_bytes/sizeof(T), sizeof(T), 0); if (rc < 0) { return filter->print_errmsg("Bit unshuffle error: " + err_msg(rc)); } return TILEDB_CDF_OK; } int CodecBitShuffle::decode(unsigned char* tile, size_t tile_size) { switch (type_) { case TILEDB_INT32: return do_decode(reinterpret_cast(tile), tile_size, this); case TILEDB_INT64: return do_decode(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT32: return do_decode(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT64: return do_decode(reinterpret_cast(tile), tile_size, this); default: return print_errmsg("CodecBitShuffle not implemented for type"); } } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_filter_delta_encode.cc000066400000000000000000000100321453617025200265540ustar00rootroot00000000000000/** * @file codec_filter_delta_encode.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the Delta Encoder Pre-Compression Filter. */ #include "codec_filter_delta_encode.h" #include "tiledb_constants.h" #include template int do_code(T* tile, size_t tile_size_in_bytes, CodecDeltaEncode* filter) { if (tile_size_in_bytes%sizeof(T) != 0) { return filter->print_errmsg("Tile size to pre-compression filter " + filter->name() + " should be a multiple of sizeof type"); } int stride = filter->stride(); size_t length = tile_size_in_bytes/sizeof(T)/stride; if (tile_size_in_bytes/sizeof(T)%stride != 0) { return filter->print_errmsg("Only tiles that are divisible by stride supported"); } std::vector last(stride, 0); for (size_t i=0; i(tile), tile_size, this); case TILEDB_INT64: return do_code(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT32: return do_code(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT64: return do_code(reinterpret_cast(tile), tile_size, this); default: return print_errmsg("CodecDeltaEncode not implemented for type"); } } template int do_decode(T* tile, size_t tile_size_in_bytes, CodecDeltaEncode* filter) { if (tile_size_in_bytes%sizeof(T) != 0) { return filter->print_errmsg("Tile size to pre-compression filter " + filter->name() + " should be a multiple of sizeof type"); } int stride = filter->stride(); size_t length = tile_size_in_bytes/sizeof(T)/stride; if (tile_size_in_bytes/sizeof(T)%stride != 0) { return filter->print_errmsg("Only tiles that are divisible by stride supported"); } std::vector last(stride, 0); for (size_t i = 0; i < length; i++) { for (int j=0; j(tile), tile_size, this); case TILEDB_INT64: return do_decode(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT32: return do_decode(reinterpret_cast(tile), tile_size, this); case TILEDB_UINT64: return do_decode(reinterpret_cast(tile), tile_size, this); default: return print_errmsg("CodecDeltaEncode not implemented for type"); } } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_gzip.cc000066400000000000000000000057711453617025200235700ustar00rootroot00000000000000/** * @file codec_gzip.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements codec for gzip for compression and decompression. */ #include "codec_gzip.h" #include "utils.h" #include int CodecGzip::do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { // Allocate space to store the compressed tile if(tile_compressed_ == NULL) { tile_compressed_allocated_size_ = tile_size + 6 + 5*(ceil(tile_size/16834.0)); tile_compressed_ = malloc(tile_compressed_allocated_size_); } // Expand compressed tile if necessary if(tile_size + 6 + 5*(ceil(tile_size/16834.0)) > tile_compressed_allocated_size_) { tile_compressed_allocated_size_ = tile_size + 6 + 5*(ceil(tile_size/16834.0)); tile_compressed_ = realloc(tile_compressed_, tile_compressed_allocated_size_); } if (tile_compressed_ == NULL) { return print_errmsg("OOM while trying to allocate memory for compress using " + name()); } // Compress tile ssize_t gzip_size = gzip(tile, tile_size, (unsigned char*)tile_compressed_, tile_compressed_allocated_size_, compression_level_); if(gzip_size == static_cast(TILEDB_UT_ERR)) { tiledb_cd_errmsg = tiledb_ut_errmsg; return TILEDB_CD_ERR; } *tile_compressed = tile_compressed_; tile_compressed_size = (size_t) gzip_size; // Success return TILEDB_CD_OK; } int CodecGzip::do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { // Decompress tile size_t gunzip_out_size; if(gunzip( tile_compressed, tile_compressed_size, tile, tile_size, gunzip_out_size) != TILEDB_UT_OK) { tiledb_cd_errmsg = tiledb_ut_errmsg; return TILEDB_CD_ERR; } // Success return TILEDB_CD_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_lz4.cc000066400000000000000000000060431453617025200233210ustar00rootroot00000000000000/** * @file codec_lz4.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements codec for LZ4 for compression and decompression. */ #include "codec_lz4.h" #include "tiledb_constants.h" #include "lz4.h" int CodecLZ4::do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { if (tile_size > LZ4_MAX_INPUT_SIZE) { return print_errmsg("Input tile size exceeds LZ4 max supported value"); } // Allocate space to store the compressed tile size_t compress_bound = LZ4_compressBound(tile_size); if(tile_compressed_ == NULL) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = malloc(compress_bound); } // Expand compressed tile if necessary if(compress_bound > tile_compressed_allocated_size_) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = realloc(tile_compressed_, compress_bound); } // Compress tile int64_t lz4_size; if (compression_level_ <= 1) { lz4_size = LZ4_compress_default((const char*)tile, (char*)tile_compressed_, tile_size, compress_bound); } else { lz4_size = LZ4_compress_fast((const char*)tile, (char*)tile_compressed_, tile_size, compress_bound, compression_level_); } if (lz4_size < 0) { return print_errmsg("Failed compressing with LZ4"); } *tile_compressed = tile_compressed_; tile_compressed_size = lz4_size; // Success return TILEDB_CD_OK; } int CodecLZ4::do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { // Decompress tile int64_t lz4_decompressed_size = LZ4_decompress_safe((const char*)tile_compressed, (char*)tile, tile_compressed_size, tile_size); if (lz4_decompressed_size < 0) { return print_errmsg("LZ4 decompression failed. lz4 error code="+std::to_string(lz4_decompressed_size)); } // Success return TILEDB_CD_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_rle.cc000066400000000000000000000110431453617025200233660ustar00rootroot00000000000000/** * @file codec_rle.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements codec for RLE for compression and decompression. */ #include "codec_rle.h" #include "utils.h" int CodecRLE::do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { // Allocate space to store the compressed tile size_t compress_bound; if(!is_coords_) compress_bound = RLE_compress_bound(tile_size, value_size_); else compress_bound = RLE_compress_bound_coords(tile_size, value_size_, dim_num_); if(tile_compressed_ == NULL) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = malloc(compress_bound); } // Expand comnpressed tile if necessary if(compress_bound > tile_compressed_allocated_size_) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = realloc(tile_compressed_, compress_bound); } // Compress tile int64_t rle_size; if(!is_coords_) { rle_size = RLE_compress( tile, tile_size, (unsigned char*) tile_compressed_, tile_compressed_allocated_size_, value_size_); } else { if(cell_order_ == TILEDB_ROW_MAJOR) { rle_size = RLE_compress_coords_row( tile, tile_size, (unsigned char*) tile_compressed_, tile_compressed_allocated_size_, value_size_, dim_num_); } else if(cell_order_ == TILEDB_COL_MAJOR) { rle_size = RLE_compress_coords_col( tile, tile_size, (unsigned char*) tile_compressed_, tile_compressed_allocated_size_, value_size_, dim_num_); } else { // Error return print_errmsg("Failed compressing with RLE; unsupported cell order"); } } // Handle error if(rle_size == TILEDB_UT_ERR) { tiledb_cd_errmsg = tiledb_ut_errmsg; return TILEDB_CD_ERR; } // Set output *tile_compressed = tile_compressed_; tile_compressed_size = (size_t) rle_size; // Success return TILEDB_CD_OK; } int CodecRLE::do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { // Decompress tile int rc; if(!is_coords_) { rc = RLE_decompress( (unsigned char*) tile_compressed, tile_compressed_size, tile, tile_size, value_size_); } else { if(cell_order_ == TILEDB_ROW_MAJOR) { rc = RLE_decompress_coords_row( (unsigned char*) tile_compressed, tile_compressed_size, tile, tile_size, value_size_, dim_num_); } else if(cell_order_ == TILEDB_COL_MAJOR) { rc = RLE_compress_coords_col( (unsigned char*) tile_compressed, tile_compressed_size, tile, tile_size, value_size_, dim_num_); } else { // Error return print_errmsg("Failed decompressing with RLE; unsupported cell order"); } } // Handle error if(rc != TILEDB_UT_OK) { tiledb_cd_errmsg = tiledb_ut_errmsg; return TILEDB_CD_ERR; } // Success return TILEDB_CD_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/codec_zstd.cc000066400000000000000000000070051453617025200235730ustar00rootroot00000000000000/** * @file codec_zstd.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements codec for Zstandard for compression and decompression. */ #ifdef ENABLE_ZSTD #define ZSTD_EXTERN_DECL extern #include "codec_zstd.h" #include int CodecZStandard::do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { // create zstd context per thread thread_local std::unique_ptr zstd_cctx(ZSTD_createCCtx(), ZSTD_freeCCtx); if (zstd_cctx.get() == nullptr) { return print_errmsg("Failed to create ZStd context for compression"); } // Allocate space to store the compressed tile size_t compress_bound = ZSTD_compressBound(tile_size); if(tile_compressed_ == NULL) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = malloc(compress_bound); } // Expand compressed tile if necessary if(compress_bound > tile_compressed_allocated_size_) { tile_compressed_allocated_size_ = compress_bound; tile_compressed_ = realloc(tile_compressed_, compress_bound); } // Compress tile size_t zstd_size = ZSTD_compressCCtx(zstd_cctx.get(), static_cast(tile_compressed_), tile_compressed_allocated_size_, // input buffer tile, tile_size, // output buffer compression_level_); if(ZSTD_isError(zstd_size)) { std::string msg = ZSTD_getErrorName(zstd_size); return print_errmsg("Failed compressing with Zstandard: " + msg); } *tile_compressed = tile_compressed_; tile_compressed_size = zstd_size; // Success return TILEDB_CD_OK; } int CodecZStandard::do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { // create zstd context per thread thread_local std::unique_ptr zstd_dctx(ZSTD_createDCtx(), ZSTD_freeCCtx); if (zstd_dctx.get() == nullptr) { return print_errmsg("Failed to create ZStd context for decompression"); } // Decompress tile size_t zstd_size = ZSTD_decompressDCtx(zstd_dctx.get(), tile, tile_size, tile_compressed, tile_compressed_size); if(ZSTD_isError(zstd_size)) { std::string msg = ZSTD_getErrorName(zstd_size); return print_errmsg("Zstandard decompression failed: " + msg); } // Success return TILEDB_CD_OK; } #else #endif /* ENABLE_ZSTD */ genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/000077500000000000000000000000001453617025200227635ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/bitshuffle/000077500000000000000000000000001453617025200251165ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/bitshuffle/bitshuffle_core.c000066400000000000000000001661161453617025200304400ustar00rootroot00000000000000/* * Bitshuffle - Filter for improving compression of typed binary data. * * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * See LICENSE file for details about copyright and rights to use. * */ #include "bitshuffle_core.h" #include "bitshuffle_internals.h" #include #include /** nalinigans@github no support for avx2/sse2 instruction as yet in this included version #if defined(__AVX2__) && defined (__SSE2__) #define USEAVX2 #endif #if defined(__SSE2__) #define USESSE2 #endif #if defined(__ARM_NEON__) || (__ARM_NEON) #define USEARMNEON #endif */ // Conditional includes for SSE2 and AVX2. #ifdef USEAVX2 #include #elif defined USESSE2 #include #elif defined USEARMNEON #include #endif #if defined(_OPENMP) && defined(_MSC_VER) typedef int64_t omp_size_t; #else typedef size_t omp_size_t; #endif // Macros. #define CHECK_MULT_EIGHT(n) if (n % 8) return -80; #define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) /* ---- Functions indicating compile time instruction set. ---- */ int bshuf_using_NEON(void) { #ifdef USEARMNEON return 1; #else return 0; #endif } int bshuf_using_SSE2(void) { #ifdef USESSE2 return 1; #else return 0; #endif } int bshuf_using_AVX2(void) { #ifdef USEAVX2 return 1; #else return 0; #endif } /* ---- Worker code not requiring special instruction sets. ---- * * The following code does not use any x86 specific vectorized instructions * and should compile on any machine * */ /* Transpose 8x8 bit array packed into a single quadword *x*. * *t* is workspace. */ #define TRANS_BIT_8X8(x, t) { \ t = (x ^ (x >> 7)) & 0x00AA00AA00AA00AALL; \ x = x ^ t ^ (t << 7); \ t = (x ^ (x >> 14)) & 0x0000CCCC0000CCCCLL; \ x = x ^ t ^ (t << 14); \ t = (x ^ (x >> 28)) & 0x00000000F0F0F0F0LL; \ x = x ^ t ^ (t << 28); \ } /* Transpose 8x8 bit array along the diagonal from upper right to lower left */ #define TRANS_BIT_8X8_BE(x, t) { \ t = (x ^ (x >> 9)) & 0x0055005500550055LL; \ x = x ^ t ^ (t << 9); \ t = (x ^ (x >> 18)) & 0x0000333300003333LL; \ x = x ^ t ^ (t << 18); \ t = (x ^ (x >> 36)) & 0x000000000F0F0F0FLL; \ x = x ^ t ^ (t << 36); \ } /* Transpose of an array of arbitrarily typed elements. */ #define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \ size_t ii, jj, kk; \ const type_t* in_type = (const type_t*) in; \ type_t* out_type = (type_t*) out; \ for(ii = 0; ii + 7 < lda; ii += 8) { \ for(jj = 0; jj < ldb; jj++) { \ for(kk = 0; kk < 8; kk++) { \ out_type[jj*lda + ii + kk] = \ in_type[ii*ldb + kk * ldb + jj]; \ } \ } \ } \ for(ii = lda - lda % 8; ii < lda; ii ++) { \ for(jj = 0; jj < ldb; jj++) { \ out_type[jj*lda + ii] = in_type[ii*ldb + jj]; \ } \ } \ } /* Memory copy with bshuf call signature. For testing and profiling. */ int64_t bshuf_copy(const void* in, void* out, const size_t size, const size_t elem_size) { const char* in_b = (const char*) in; char* out_b = (char*) out; memcpy(out_b, in_b, size * elem_size); return size * elem_size; } /* Transpose bytes within elements, starting partway through input. */ int64_t bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size, const size_t elem_size, const size_t start) { size_t ii, jj, kk; const char* in_b = (const char*) in; char* out_b = (char*) out; CHECK_MULT_EIGHT(start); if (size > start) { // ii loop separated into 2 loops so the compiler can unroll // the inner one. for (ii = start; ii + 7 < size; ii += 8) { for (jj = 0; jj < elem_size; jj++) { for (kk = 0; kk < 8; kk++) { out_b[jj * size + ii + kk] = in_b[ii * elem_size + kk * elem_size + jj]; } } } for (ii = size - size % 8; ii < size; ii ++) { for (jj = 0; jj < elem_size; jj++) { out_b[jj * size + ii] = in_b[ii * elem_size + jj]; } } } return size * elem_size; } /* Transpose bytes within elements. */ int64_t bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size, const size_t elem_size) { return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0); } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size, const size_t elem_size, const size_t start_byte) { const uint64_t* in_b = (const uint64_t*) in; uint8_t* out_b = (uint8_t*) out; uint64_t x, t; size_t ii, kk; size_t nbyte = elem_size * size; size_t nbyte_bitrow = nbyte / 8; uint64_t e=1; const int little_endian = *(uint8_t *) &e == 1; const size_t bit_row_skip = little_endian ? nbyte_bitrow : -nbyte_bitrow; const int64_t bit_row_offset = little_endian ? 0 : 7 * nbyte_bitrow; CHECK_MULT_EIGHT(nbyte); CHECK_MULT_EIGHT(start_byte); for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) { x = in_b[ii]; if (little_endian) { TRANS_BIT_8X8(x, t); } else { TRANS_BIT_8X8_BE(x, t); } for (kk = 0; kk < 8; kk ++) { out_b[bit_row_offset + kk * bit_row_skip + ii] = x; x = x >> 8; } } return size * elem_size; } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_scal(const void* in, void* out, const size_t size, const size_t elem_size) { return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0); } /* General transpose of an array, optimized for large element sizes. */ int64_t bshuf_trans_elem(const void* in, void* out, const size_t lda, const size_t ldb, const size_t elem_size) { size_t ii, jj; const char* in_b = (const char*) in; char* out_b = (char*) out; for(ii = 0; ii < lda; ii++) { for(jj = 0; jj < ldb; jj++) { memcpy(&out_b[(jj*lda + ii) * elem_size], &in_b[(ii*ldb + jj) * elem_size], elem_size); } } return lda * ldb * elem_size; } /* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */ int64_t bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size, const size_t elem_size) { size_t nbyte_bitrow = size / 8; CHECK_MULT_EIGHT(size); return bshuf_trans_elem(in, out, 8, elem_size, nbyte_bitrow); } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; void *tmp_buf; CHECK_MULT_EIGHT(size); tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bit_byte_scal(out, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size, const size_t elem_size) { size_t ii, jj, kk, nbyte_row; const char *in_b; char *out_b; in_b = (const char*) in; out_b = (char*) out; nbyte_row = size / 8; CHECK_MULT_EIGHT(size); for (jj = 0; jj < elem_size; jj++) { for (ii = 0; ii < nbyte_row; ii++) { for (kk = 0; kk < 8; kk++) { out_b[ii * 8 * elem_size + jj * 8 + kk] = \ in_b[(jj * 8 + kk) * nbyte_row + ii]; } } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \ const size_t size, const size_t elem_size) { const char *in_b; char *out_b; uint64_t x, t; size_t ii, jj, kk; size_t nbyte, out_index; uint64_t e=1; const int little_endian = *(uint8_t *) &e == 1; const size_t elem_skip = little_endian ? elem_size : -elem_size; const uint64_t elem_offset = little_endian ? 0 : 7 * elem_size; CHECK_MULT_EIGHT(size); in_b = (const char*) in; out_b = (char*) out; nbyte = elem_size * size; for (jj = 0; jj < 8 * elem_size; jj += 8) { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { x = *((uint64_t*) &in_b[ii + jj]); if (little_endian) { TRANS_BIT_8X8(x, t); } else { TRANS_BIT_8X8_BE(x, t); } for (kk = 0; kk < 8; kk++) { out_index = ii + jj / 8 + elem_offset + kk * elem_skip; *((uint8_t*) &out_b[out_index]) = x; x = x >> 8; } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; void *tmp_buf; CHECK_MULT_EIGHT(size); tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_bitrow_scal(in, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_shuffle_bit_eightelem_scal(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } /* ---- Worker code that uses Arm NEON ---- * * The following code makes use of the Arm NEON instruction set. * NEON technology is the implementation of the ARM Advanced Single * Instruction Multiple Data (SIMD) extension. * The NEON unit is the component of the processor that executes SIMD instructions. * It is also called the NEON Media Processing Engine (MPE). * */ #ifdef USEARMNEON /* Transpose bytes within elements for 16 bit elements. */ int64_t bshuf_trans_byte_elem_NEON_16(const void* in, void* out, const size_t size) { size_t ii; const char *in_b = (const char*) in; char *out_b = (char*) out; int8x16_t a0, b0, a1, b1; for (ii=0; ii + 15 < size; ii += 16) { a0 = vld1q_s8(in_b + 2*ii + 0*16); b0 = vld1q_s8(in_b + 2*ii + 1*16); a1 = vzip1q_s8(a0, b0); b1 = vzip2q_s8(a0, b0); a0 = vzip1q_s8(a1, b1); b0 = vzip2q_s8(a1, b1); a1 = vzip1q_s8(a0, b0); b1 = vzip2q_s8(a0, b0); a0 = vzip1q_s8(a1, b1); b0 = vzip2q_s8(a1, b1); vst1q_s8(out_b + 0*size + ii, a0); vst1q_s8(out_b + 1*size + ii, b0); } return bshuf_trans_byte_elem_remainder(in, out, size, 2, size - size % 16); } /* Transpose bytes within elements for 32 bit elements. */ int64_t bshuf_trans_byte_elem_NEON_32(const void* in, void* out, const size_t size) { size_t ii; const char *in_b; char *out_b; in_b = (const char*) in; out_b = (char*) out; int8x16_t a0, b0, c0, d0, a1, b1, c1, d1; int64x2_t a2, b2, c2, d2; for (ii=0; ii + 15 < size; ii += 16) { a0 = vld1q_s8(in_b + 4*ii + 0*16); b0 = vld1q_s8(in_b + 4*ii + 1*16); c0 = vld1q_s8(in_b + 4*ii + 2*16); d0 = vld1q_s8(in_b + 4*ii + 3*16); a1 = vzip1q_s8(a0, b0); b1 = vzip2q_s8(a0, b0); c1 = vzip1q_s8(c0, d0); d1 = vzip2q_s8(c0, d0); a0 = vzip1q_s8(a1, b1); b0 = vzip2q_s8(a1, b1); c0 = vzip1q_s8(c1, d1); d0 = vzip2q_s8(c1, d1); a1 = vzip1q_s8(a0, b0); b1 = vzip2q_s8(a0, b0); c1 = vzip1q_s8(c0, d0); d1 = vzip2q_s8(c0, d0); a2 = vzip1q_s64(vreinterpretq_s64_s8(a1), vreinterpretq_s64_s8(c1)); b2 = vzip2q_s64(vreinterpretq_s64_s8(a1), vreinterpretq_s64_s8(c1)); c2 = vzip1q_s64(vreinterpretq_s64_s8(b1), vreinterpretq_s64_s8(d1)); d2 = vzip2q_s64(vreinterpretq_s64_s8(b1), vreinterpretq_s64_s8(d1)); vst1q_s64((int64_t *) (out_b + 0*size + ii), a2); vst1q_s64((int64_t *) (out_b + 1*size + ii), b2); vst1q_s64((int64_t *) (out_b + 2*size + ii), c2); vst1q_s64((int64_t *) (out_b + 3*size + ii), d2); } return bshuf_trans_byte_elem_remainder(in, out, size, 4, size - size % 16); } /* Transpose bytes within elements for 64 bit elements. */ int64_t bshuf_trans_byte_elem_NEON_64(const void* in, void* out, const size_t size) { size_t ii; const char* in_b = (const char*) in; char* out_b = (char*) out; int8x16_t a0, b0, c0, d0, e0, f0, g0, h0; int8x16_t a1, b1, c1, d1, e1, f1, g1, h1; for (ii=0; ii + 15 < size; ii += 16) { a0 = vld1q_s8(in_b + 8*ii + 0*16); b0 = vld1q_s8(in_b + 8*ii + 1*16); c0 = vld1q_s8(in_b + 8*ii + 2*16); d0 = vld1q_s8(in_b + 8*ii + 3*16); e0 = vld1q_s8(in_b + 8*ii + 4*16); f0 = vld1q_s8(in_b + 8*ii + 5*16); g0 = vld1q_s8(in_b + 8*ii + 6*16); h0 = vld1q_s8(in_b + 8*ii + 7*16); a1 = vzip1q_s8 (a0, b0); b1 = vzip2q_s8 (a0, b0); c1 = vzip1q_s8 (c0, d0); d1 = vzip2q_s8 (c0, d0); e1 = vzip1q_s8 (e0, f0); f1 = vzip2q_s8 (e0, f0); g1 = vzip1q_s8 (g0, h0); h1 = vzip2q_s8 (g0, h0); a0 = vzip1q_s8 (a1, b1); b0 = vzip2q_s8 (a1, b1); c0 = vzip1q_s8 (c1, d1); d0 = vzip2q_s8 (c1, d1); e0 = vzip1q_s8 (e1, f1); f0 = vzip2q_s8 (e1, f1); g0 = vzip1q_s8 (g1, h1); h0 = vzip2q_s8 (g1, h1); a1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (a0), vreinterpretq_s32_s8 (c0)); b1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (a0), vreinterpretq_s32_s8 (c0)); c1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (b0), vreinterpretq_s32_s8 (d0)); d1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (b0), vreinterpretq_s32_s8 (d0)); e1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (e0), vreinterpretq_s32_s8 (g0)); f1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (e0), vreinterpretq_s32_s8 (g0)); g1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (f0), vreinterpretq_s32_s8 (h0)); h1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (f0), vreinterpretq_s32_s8 (h0)); a0 = (int8x16_t) vzip1q_s64 (vreinterpretq_s64_s8 (a1), vreinterpretq_s64_s8 (e1)); b0 = (int8x16_t) vzip2q_s64 (vreinterpretq_s64_s8 (a1), vreinterpretq_s64_s8 (e1)); c0 = (int8x16_t) vzip1q_s64 (vreinterpretq_s64_s8 (b1), vreinterpretq_s64_s8 (f1)); d0 = (int8x16_t) vzip2q_s64 (vreinterpretq_s64_s8 (b1), vreinterpretq_s64_s8 (f1)); e0 = (int8x16_t) vzip1q_s64 (vreinterpretq_s64_s8 (c1), vreinterpretq_s64_s8 (g1)); f0 = (int8x16_t) vzip2q_s64 (vreinterpretq_s64_s8 (c1), vreinterpretq_s64_s8 (g1)); g0 = (int8x16_t) vzip1q_s64 (vreinterpretq_s64_s8 (d1), vreinterpretq_s64_s8 (h1)); h0 = (int8x16_t) vzip2q_s64 (vreinterpretq_s64_s8 (d1), vreinterpretq_s64_s8 (h1)); vst1q_s8(out_b + 0*size + ii, a0); vst1q_s8(out_b + 1*size + ii, b0); vst1q_s8(out_b + 2*size + ii, c0); vst1q_s8(out_b + 3*size + ii, d0); vst1q_s8(out_b + 4*size + ii, e0); vst1q_s8(out_b + 5*size + ii, f0); vst1q_s8(out_b + 6*size + ii, g0); vst1q_s8(out_b + 7*size + ii, h0); } return bshuf_trans_byte_elem_remainder(in, out, size, 8, size - size % 16); } /* Transpose bytes within elements using best NEON algorithm available. */ int64_t bshuf_trans_byte_elem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; // Trivial cases: power of 2 bytes. switch (elem_size) { case 1: count = bshuf_copy(in, out, size, elem_size); return count; case 2: count = bshuf_trans_byte_elem_NEON_16(in, out, size); return count; case 4: count = bshuf_trans_byte_elem_NEON_32(in, out, size); return count; case 8: count = bshuf_trans_byte_elem_NEON_64(in, out, size); return count; } // Worst case: odd number of bytes. Turns out that this is faster for // (odd * 2) byte elements as well (hence % 4). if (elem_size % 4) { count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); return count; } // Multiple of power of 2: transpose hierarchically. { size_t nchunk_elem; void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; if ((elem_size % 8) == 0) { nchunk_elem = elem_size / 8; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t); count = bshuf_trans_byte_elem_NEON_64(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size); } else if ((elem_size % 4) == 0) { nchunk_elem = elem_size / 4; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t); count = bshuf_trans_byte_elem_NEON_32(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size); } else { // Not used since scalar algorithm is faster. nchunk_elem = elem_size / 2; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t); count = bshuf_trans_byte_elem_NEON_16(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size); } free(tmp_buf); return count; } } /* Creates a mask made up of the most significant * bit of each byte of 'input' */ int32_t move_byte_mask_neon(uint8x16_t input) { return ( ((input[0] & 0x80) >> 7) | (((input[1] & 0x80) >> 7) << 1) | (((input[2] & 0x80) >> 7) << 2) | (((input[3] & 0x80) >> 7) << 3) | (((input[4] & 0x80) >> 7) << 4) | (((input[5] & 0x80) >> 7) << 5) | (((input[6] & 0x80) >> 7) << 6) | (((input[7] & 0x80) >> 7) << 7) | (((input[8] & 0x80) >> 7) << 8) | (((input[9] & 0x80) >> 7) << 9) | (((input[10] & 0x80) >> 7) << 10) | (((input[11] & 0x80) >> 7) << 11) | (((input[12] & 0x80) >> 7) << 12) | (((input[13] & 0x80) >> 7) << 13) | (((input[14] & 0x80) >> 7) << 14) | (((input[15] & 0x80) >> 7) << 15) ); } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { size_t ii, kk; const char* in_b = (const char*) in; char* out_b = (char*) out; uint16_t* out_ui16; int64_t count; size_t nbyte = elem_size * size; CHECK_MULT_EIGHT(nbyte); int16x8_t xmm; int32_t bt; for (ii = 0; ii + 15 < nbyte; ii += 16) { xmm = vld1q_s16((int16_t *) (in_b + ii)); for (kk = 0; kk < 8; kk++) { bt = move_byte_mask_neon((uint8x16_t) xmm); xmm = vshlq_n_s16(xmm, 1); out_ui16 = (uint16_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; *out_ui16 = bt; } } count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, nbyte - nbyte % 16); return count; } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; CHECK_MULT_EIGHT(size); void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_elem_NEON(in, out, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bit_byte_NEON(out, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { size_t ii, jj; const char* in_b = (const char*) in; char* out_b = (char*) out; CHECK_MULT_EIGHT(size); size_t nrows = 8 * elem_size; size_t nbyte_row = size / 8; int8x16_t a0, b0, c0, d0, e0, f0, g0, h0; int8x16_t a1, b1, c1, d1, e1, f1, g1, h1; int64x1_t *as, *bs, *cs, *ds, *es, *fs, *gs, *hs; for (ii = 0; ii + 7 < nrows; ii += 8) { for (jj = 0; jj + 15 < nbyte_row; jj += 16) { a0 = vld1q_s8(in_b + (ii + 0)*nbyte_row + jj); b0 = vld1q_s8(in_b + (ii + 1)*nbyte_row + jj); c0 = vld1q_s8(in_b + (ii + 2)*nbyte_row + jj); d0 = vld1q_s8(in_b + (ii + 3)*nbyte_row + jj); e0 = vld1q_s8(in_b + (ii + 4)*nbyte_row + jj); f0 = vld1q_s8(in_b + (ii + 5)*nbyte_row + jj); g0 = vld1q_s8(in_b + (ii + 6)*nbyte_row + jj); h0 = vld1q_s8(in_b + (ii + 7)*nbyte_row + jj); a1 = vzip1q_s8(a0, b0); b1 = vzip1q_s8(c0, d0); c1 = vzip1q_s8(e0, f0); d1 = vzip1q_s8(g0, h0); e1 = vzip2q_s8(a0, b0); f1 = vzip2q_s8(c0, d0); g1 = vzip2q_s8(e0, f0); h1 = vzip2q_s8(g0, h0); a0 = (int8x16_t) vzip1q_s16 (vreinterpretq_s16_s8 (a1), vreinterpretq_s16_s8 (b1)); b0= (int8x16_t) vzip1q_s16 (vreinterpretq_s16_s8 (c1), vreinterpretq_s16_s8 (d1)); c0 = (int8x16_t) vzip2q_s16 (vreinterpretq_s16_s8 (a1), vreinterpretq_s16_s8 (b1)); d0 = (int8x16_t) vzip2q_s16 (vreinterpretq_s16_s8 (c1), vreinterpretq_s16_s8 (d1)); e0 = (int8x16_t) vzip1q_s16 (vreinterpretq_s16_s8 (e1), vreinterpretq_s16_s8 (f1)); f0 = (int8x16_t) vzip1q_s16 (vreinterpretq_s16_s8 (g1), vreinterpretq_s16_s8 (h1)); g0 = (int8x16_t) vzip2q_s16 (vreinterpretq_s16_s8 (e1), vreinterpretq_s16_s8 (f1)); h0 = (int8x16_t) vzip2q_s16 (vreinterpretq_s16_s8 (g1), vreinterpretq_s16_s8 (h1)); a1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (a0), vreinterpretq_s32_s8 (b0)); b1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (a0), vreinterpretq_s32_s8 (b0)); c1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (c0), vreinterpretq_s32_s8 (d0)); d1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (c0), vreinterpretq_s32_s8 (d0)); e1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (e0), vreinterpretq_s32_s8 (f0)); f1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (e0), vreinterpretq_s32_s8 (f0)); g1 = (int8x16_t) vzip1q_s32 (vreinterpretq_s32_s8 (g0), vreinterpretq_s32_s8 (h0)); h1 = (int8x16_t) vzip2q_s32 (vreinterpretq_s32_s8 (g0), vreinterpretq_s32_s8 (h0)); as = (int64x1_t *) &a1; bs = (int64x1_t *) &b1; cs = (int64x1_t *) &c1; ds = (int64x1_t *) &d1; es = (int64x1_t *) &e1; fs = (int64x1_t *) &f1; gs = (int64x1_t *) &g1; hs = (int64x1_t *) &h1; vst1_s64((int64_t *)(out_b + (jj + 0) * nrows + ii), *as); vst1_s64((int64_t *)(out_b + (jj + 1) * nrows + ii), *(as + 1)); vst1_s64((int64_t *)(out_b + (jj + 2) * nrows + ii), *bs); vst1_s64((int64_t *)(out_b + (jj + 3) * nrows + ii), *(bs + 1)); vst1_s64((int64_t *)(out_b + (jj + 4) * nrows + ii), *cs); vst1_s64((int64_t *)(out_b + (jj + 5) * nrows + ii), *(cs + 1)); vst1_s64((int64_t *)(out_b + (jj + 6) * nrows + ii), *ds); vst1_s64((int64_t *)(out_b + (jj + 7) * nrows + ii), *(ds + 1)); vst1_s64((int64_t *)(out_b + (jj + 8) * nrows + ii), *es); vst1_s64((int64_t *)(out_b + (jj + 9) * nrows + ii), *(es + 1)); vst1_s64((int64_t *)(out_b + (jj + 10) * nrows + ii), *fs); vst1_s64((int64_t *)(out_b + (jj + 11) * nrows + ii), *(fs + 1)); vst1_s64((int64_t *)(out_b + (jj + 12) * nrows + ii), *gs); vst1_s64((int64_t *)(out_b + (jj + 13) * nrows + ii), *(gs + 1)); vst1_s64((int64_t *)(out_b + (jj + 14) * nrows + ii), *hs); vst1_s64((int64_t *)(out_b + (jj + 15) * nrows + ii), *(hs + 1)); } for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) { out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj]; out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj]; out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj]; out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj]; out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj]; out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj]; out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj]; out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj]; } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { CHECK_MULT_EIGHT(size); // With a bit of care, this could be written such that such that it is // in_buf = out_buf safe. const char* in_b = (const char*) in; uint16_t* out_ui16 = (uint16_t*) out; size_t ii, jj, kk; size_t nbyte = elem_size * size; int16x8_t xmm; int32_t bt; if (elem_size % 2) { bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); } else { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) { xmm = vld1q_s16((int16_t *) &in_b[ii + jj]); for (kk = 0; kk < 8; kk++) { bt = move_byte_mask_neon((uint8x16_t) xmm); xmm = vshlq_n_s16(xmm, 1); size_t ind = (ii + jj / 8 + (7 - kk) * elem_size); out_ui16[ind / 2] = bt; } } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; CHECK_MULT_EIGHT(size); void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_bitrow_NEON(in, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_shuffle_bit_eightelem_NEON(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } #else // #ifdef USEARMNEON int64_t bshuf_untrans_bit_elem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { return -13; } int64_t bshuf_trans_bit_elem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { return -13; } int64_t bshuf_trans_byte_bitrow_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { return -13; } int64_t bshuf_trans_bit_byte_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { return -13; } int64_t bshuf_trans_byte_elem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { return -13; } int64_t bshuf_trans_byte_elem_NEON_64(const void* in, void* out, const size_t size) { return -13; } int64_t bshuf_trans_byte_elem_NEON_32(const void* in, void* out, const size_t size) { return -13; } int64_t bshuf_trans_byte_elem_NEON_16(const void* in, void* out, const size_t size) { return -13; } int64_t bshuf_shuffle_bit_eightelem_NEON(const void* in, void* out, const size_t size, const size_t elem_size) { return -13; } #endif /* ---- Worker code that uses SSE2 ---- * * The following code makes use of the SSE2 instruction set and specialized * 16 byte registers. The SSE2 instructions are present on modern x86 * processors. The first Intel processor microarchitecture supporting SSE2 was * Pentium 4 (2000). * */ #ifdef USESSE2 /* Transpose bytes within elements for 16 bit elements. */ int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) { size_t ii; const char *in_b = (const char*) in; char *out_b = (char*) out; __m128i a0, b0, a1, b1; for (ii=0; ii + 15 < size; ii += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 0*16]); b0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 1*16]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); } return bshuf_trans_byte_elem_remainder(in, out, size, 2, size - size % 16); } /* Transpose bytes within elements for 32 bit elements. */ int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) { size_t ii; const char *in_b; char *out_b; in_b = (const char*) in; out_b = (char*) out; __m128i a0, b0, c0, d0, a1, b1, c1, d1; for (ii=0; ii + 15 < size; ii += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 0*16]); b0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 1*16]); c0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 2*16]); d0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 3*16]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); c1 = _mm_unpacklo_epi8(c0, d0); d1 = _mm_unpackhi_epi8(c0, d0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); c0 = _mm_unpacklo_epi8(c1, d1); d0 = _mm_unpackhi_epi8(c1, d1); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); c1 = _mm_unpacklo_epi8(c0, d0); d1 = _mm_unpackhi_epi8(c0, d0); a0 = _mm_unpacklo_epi64(a1, c1); b0 = _mm_unpackhi_epi64(a1, c1); c0 = _mm_unpacklo_epi64(b1, d1); d0 = _mm_unpackhi_epi64(b1, d1); _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); } return bshuf_trans_byte_elem_remainder(in, out, size, 4, size - size % 16); } /* Transpose bytes within elements for 64 bit elements. */ int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) { size_t ii; const char* in_b = (const char*) in; char* out_b = (char*) out; __m128i a0, b0, c0, d0, e0, f0, g0, h0; __m128i a1, b1, c1, d1, e1, f1, g1, h1; for (ii=0; ii + 15 < size; ii += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]); b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]); c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]); d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]); e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]); f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]); g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]); h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); c1 = _mm_unpacklo_epi8(c0, d0); d1 = _mm_unpackhi_epi8(c0, d0); e1 = _mm_unpacklo_epi8(e0, f0); f1 = _mm_unpackhi_epi8(e0, f0); g1 = _mm_unpacklo_epi8(g0, h0); h1 = _mm_unpackhi_epi8(g0, h0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); c0 = _mm_unpacklo_epi8(c1, d1); d0 = _mm_unpackhi_epi8(c1, d1); e0 = _mm_unpacklo_epi8(e1, f1); f0 = _mm_unpackhi_epi8(e1, f1); g0 = _mm_unpacklo_epi8(g1, h1); h0 = _mm_unpackhi_epi8(g1, h1); a1 = _mm_unpacklo_epi32(a0, c0); b1 = _mm_unpackhi_epi32(a0, c0); c1 = _mm_unpacklo_epi32(b0, d0); d1 = _mm_unpackhi_epi32(b0, d0); e1 = _mm_unpacklo_epi32(e0, g0); f1 = _mm_unpackhi_epi32(e0, g0); g1 = _mm_unpacklo_epi32(f0, h0); h1 = _mm_unpackhi_epi32(f0, h0); a0 = _mm_unpacklo_epi64(a1, e1); b0 = _mm_unpackhi_epi64(a1, e1); c0 = _mm_unpacklo_epi64(b1, f1); d0 = _mm_unpackhi_epi64(b1, f1); e0 = _mm_unpacklo_epi64(c1, g1); f0 = _mm_unpackhi_epi64(c1, g1); g0 = _mm_unpacklo_epi64(d1, h1); h0 = _mm_unpackhi_epi64(d1, h1); _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); _mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0); _mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0); _mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0); _mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0); } return bshuf_trans_byte_elem_remainder(in, out, size, 8, size - size % 16); } /* Transpose bytes within elements using best SSE algorithm available. */ int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; // Trivial cases: power of 2 bytes. switch (elem_size) { case 1: count = bshuf_copy(in, out, size, elem_size); return count; case 2: count = bshuf_trans_byte_elem_SSE_16(in, out, size); return count; case 4: count = bshuf_trans_byte_elem_SSE_32(in, out, size); return count; case 8: count = bshuf_trans_byte_elem_SSE_64(in, out, size); return count; } // Worst case: odd number of bytes. Turns out that this is faster for // (odd * 2) byte elements as well (hence % 4). if (elem_size % 4) { count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); return count; } // Multiple of power of 2: transpose hierarchically. { size_t nchunk_elem; void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; if ((elem_size % 8) == 0) { nchunk_elem = elem_size / 8; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t); count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size); } else if ((elem_size % 4) == 0) { nchunk_elem = elem_size / 4; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t); count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size); } else { // Not used since scalar algorithm is faster. nchunk_elem = elem_size / 2; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t); count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size); } free(tmp_buf); return count; } } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { size_t ii, kk; const char* in_b = (const char*) in; char* out_b = (char*) out; uint16_t* out_ui16; int64_t count; size_t nbyte = elem_size * size; CHECK_MULT_EIGHT(nbyte); __m128i xmm; int32_t bt; for (ii = 0; ii + 15 < nbyte; ii += 16) { xmm = _mm_loadu_si128((__m128i *) &in_b[ii]); for (kk = 0; kk < 8; kk++) { bt = _mm_movemask_epi8(xmm); xmm = _mm_slli_epi16(xmm, 1); out_ui16 = (uint16_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; *out_ui16 = bt; } } count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, nbyte - nbyte % 16); return count; } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; CHECK_MULT_EIGHT(size); void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bit_byte_SSE(out, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { size_t ii, jj; const char* in_b = (const char*) in; char* out_b = (char*) out; CHECK_MULT_EIGHT(size); size_t nrows = 8 * elem_size; size_t nbyte_row = size / 8; __m128i a0, b0, c0, d0, e0, f0, g0, h0; __m128i a1, b1, c1, d1, e1, f1, g1, h1; __m128 *as, *bs, *cs, *ds, *es, *fs, *gs, *hs; for (ii = 0; ii + 7 < nrows; ii += 8) { for (jj = 0; jj + 15 < nbyte_row; jj += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 0)*nbyte_row + jj]); b0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 1)*nbyte_row + jj]); c0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 2)*nbyte_row + jj]); d0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 3)*nbyte_row + jj]); e0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 4)*nbyte_row + jj]); f0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 5)*nbyte_row + jj]); g0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 6)*nbyte_row + jj]); h0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 7)*nbyte_row + jj]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpacklo_epi8(c0, d0); c1 = _mm_unpacklo_epi8(e0, f0); d1 = _mm_unpacklo_epi8(g0, h0); e1 = _mm_unpackhi_epi8(a0, b0); f1 = _mm_unpackhi_epi8(c0, d0); g1 = _mm_unpackhi_epi8(e0, f0); h1 = _mm_unpackhi_epi8(g0, h0); a0 = _mm_unpacklo_epi16(a1, b1); b0 = _mm_unpacklo_epi16(c1, d1); c0 = _mm_unpackhi_epi16(a1, b1); d0 = _mm_unpackhi_epi16(c1, d1); e0 = _mm_unpacklo_epi16(e1, f1); f0 = _mm_unpacklo_epi16(g1, h1); g0 = _mm_unpackhi_epi16(e1, f1); h0 = _mm_unpackhi_epi16(g1, h1); a1 = _mm_unpacklo_epi32(a0, b0); b1 = _mm_unpackhi_epi32(a0, b0); c1 = _mm_unpacklo_epi32(c0, d0); d1 = _mm_unpackhi_epi32(c0, d0); e1 = _mm_unpacklo_epi32(e0, f0); f1 = _mm_unpackhi_epi32(e0, f0); g1 = _mm_unpacklo_epi32(g0, h0); h1 = _mm_unpackhi_epi32(g0, h0); // We don't have a storeh instruction for integers, so interpret // as a float. Have a storel (_mm_storel_epi64). as = (__m128 *) &a1; bs = (__m128 *) &b1; cs = (__m128 *) &c1; ds = (__m128 *) &d1; es = (__m128 *) &e1; fs = (__m128 *) &f1; gs = (__m128 *) &g1; hs = (__m128 *) &h1; _mm_storel_pi((__m64 *) &out_b[(jj + 0) * nrows + ii], *as); _mm_storel_pi((__m64 *) &out_b[(jj + 2) * nrows + ii], *bs); _mm_storel_pi((__m64 *) &out_b[(jj + 4) * nrows + ii], *cs); _mm_storel_pi((__m64 *) &out_b[(jj + 6) * nrows + ii], *ds); _mm_storel_pi((__m64 *) &out_b[(jj + 8) * nrows + ii], *es); _mm_storel_pi((__m64 *) &out_b[(jj + 10) * nrows + ii], *fs); _mm_storel_pi((__m64 *) &out_b[(jj + 12) * nrows + ii], *gs); _mm_storel_pi((__m64 *) &out_b[(jj + 14) * nrows + ii], *hs); _mm_storeh_pi((__m64 *) &out_b[(jj + 1) * nrows + ii], *as); _mm_storeh_pi((__m64 *) &out_b[(jj + 3) * nrows + ii], *bs); _mm_storeh_pi((__m64 *) &out_b[(jj + 5) * nrows + ii], *cs); _mm_storeh_pi((__m64 *) &out_b[(jj + 7) * nrows + ii], *ds); _mm_storeh_pi((__m64 *) &out_b[(jj + 9) * nrows + ii], *es); _mm_storeh_pi((__m64 *) &out_b[(jj + 11) * nrows + ii], *fs); _mm_storeh_pi((__m64 *) &out_b[(jj + 13) * nrows + ii], *gs); _mm_storeh_pi((__m64 *) &out_b[(jj + 15) * nrows + ii], *hs); } for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) { out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj]; out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj]; out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj]; out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj]; out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj]; out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj]; out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj]; out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj]; } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { CHECK_MULT_EIGHT(size); // With a bit of care, this could be written such that such that it is // in_buf = out_buf safe. const char* in_b = (const char*) in; uint16_t* out_ui16 = (uint16_t*) out; size_t ii, jj, kk; size_t nbyte = elem_size * size; __m128i xmm; int32_t bt; if (elem_size % 2) { bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); } else { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) { xmm = _mm_loadu_si128((__m128i *) &in_b[ii + jj]); for (kk = 0; kk < 8; kk++) { bt = _mm_movemask_epi8(xmm); xmm = _mm_slli_epi16(xmm, 1); size_t ind = (ii + jj / 8 + (7 - kk) * elem_size); out_ui16[ind / 2] = bt; } } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; CHECK_MULT_EIGHT(size); void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_bitrow_SSE(in, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_shuffle_bit_eightelem_SSE(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } #else // #ifdef USESSE2 int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { return -11; } int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { return -11; } int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { return -11; } int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { return -11; } int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { return -11; } int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) { return -11; } int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) { return -11; } int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) { return -11; } int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size, const size_t elem_size) { return -11; } #endif // #ifdef USESSE2 /* ---- Code that requires AVX2. Intel Haswell (2013) and later. ---- */ /* ---- Worker code that uses AVX2 ---- * * The following code makes use of the AVX2 instruction set and specialized * 32 byte registers. The AVX2 instructions are present on newer x86 * processors. The first Intel processor microarchitecture supporting AVX2 was * Haswell (2013). * */ #ifdef USEAVX2 /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { size_t ii, kk; const char* in_b = (const char*) in; char* out_b = (char*) out; int32_t* out_i32; size_t nbyte = elem_size * size; int64_t count; __m256i ymm; int32_t bt; for (ii = 0; ii + 31 < nbyte; ii += 32) { ymm = _mm256_loadu_si256((__m256i *) &in_b[ii]); for (kk = 0; kk < 8; kk++) { bt = _mm256_movemask_epi8(ymm); ymm = _mm256_slli_epi16(ymm, 1); out_i32 = (int32_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; *out_i32 = bt; } } count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, nbyte - nbyte % 32); return count; } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; CHECK_MULT_EIGHT(size); void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bit_byte_AVX(out, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { size_t hh, ii, jj, kk, mm; const char* in_b = (const char*) in; char* out_b = (char*) out; CHECK_MULT_EIGHT(size); size_t nrows = 8 * elem_size; size_t nbyte_row = size / 8; if (elem_size % 4) return bshuf_trans_byte_bitrow_SSE(in, out, size, elem_size); __m256i ymm_0[8]; __m256i ymm_1[8]; __m256i ymm_storeage[8][4]; for (jj = 0; jj + 31 < nbyte_row; jj += 32) { for (ii = 0; ii + 3 < elem_size; ii += 4) { for (hh = 0; hh < 4; hh ++) { for (kk = 0; kk < 8; kk ++){ ymm_0[kk] = _mm256_loadu_si256((__m256i *) &in_b[ (ii * 8 + hh * 8 + kk) * nbyte_row + jj]); } for (kk = 0; kk < 4; kk ++){ ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); } for (kk = 0; kk < 2; kk ++){ for (mm = 0; mm < 2; mm ++){ ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16( ymm_1[kk * 4 + mm * 2], ymm_1[kk * 4 + mm * 2 + 1]); ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16( ymm_1[kk * 4 + mm * 2], ymm_1[kk * 4 + mm * 2 + 1]); } } for (kk = 0; kk < 4; kk ++){ ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); } for (kk = 0; kk < 8; kk ++){ ymm_storeage[kk][hh] = ymm_1[kk]; } } for (mm = 0; mm < 8; mm ++) { for (kk = 0; kk < 4; kk ++){ ymm_0[kk] = ymm_storeage[mm][kk]; } ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]); ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]); ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]); ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]); ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32); ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32); ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49); ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]); } } } for (ii = 0; ii < nrows; ii ++ ) { for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj ++) { out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj]; } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { CHECK_MULT_EIGHT(size); // With a bit of care, this could be written such that such that it is // in_buf = out_buf safe. const char* in_b = (const char*) in; char* out_b = (char*) out; size_t ii, jj, kk; size_t nbyte = elem_size * size; __m256i ymm; int32_t bt; if (elem_size % 4) { return bshuf_shuffle_bit_eightelem_SSE(in, out, size, elem_size); } else { for (jj = 0; jj + 31 < 8 * elem_size; jj += 32) { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { ymm = _mm256_loadu_si256((__m256i *) &in_b[ii + jj]); for (kk = 0; kk < 8; kk++) { bt = _mm256_movemask_epi8(ymm); ymm = _mm256_slli_epi16(ymm, 1); size_t ind = (ii + jj / 8 + (7 - kk) * elem_size); * (int32_t *) &out_b[ind] = bt; } } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; CHECK_MULT_EIGHT(size); void* tmp_buf = malloc(size * elem_size); if (tmp_buf == NULL) return -1; count = bshuf_trans_byte_bitrow_AVX(in, tmp_buf, size, elem_size); CHECK_ERR_FREE(count, tmp_buf); count = bshuf_shuffle_bit_eightelem_AVX(tmp_buf, out, size, elem_size); free(tmp_buf); return count; } #else // #ifdef USEAVX2 int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { return -12; } int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { return -12; } int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { return -12; } int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { return -12; } int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size, const size_t elem_size) { return -12; } #endif // #ifdef USEAVX2 /* ---- Drivers selecting best instruction set at compile time. ---- */ int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; #ifdef USEAVX2 count = bshuf_trans_bit_elem_AVX(in, out, size, elem_size); #elif defined(USESSE2) count = bshuf_trans_bit_elem_SSE(in, out, size, elem_size); #elif defined(USEARMNEON) count = bshuf_trans_bit_elem_NEON(in, out, size, elem_size); #else count = bshuf_trans_bit_elem_scal(in, out, size, elem_size); #endif return count; } int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size, const size_t elem_size) { int64_t count; #ifdef USEAVX2 count = bshuf_untrans_bit_elem_AVX(in, out, size, elem_size); #elif defined(USESSE2) count = bshuf_untrans_bit_elem_SSE(in, out, size, elem_size); #elif defined(USEARMNEON) count = bshuf_untrans_bit_elem_NEON(in, out, size, elem_size); #else count = bshuf_untrans_bit_elem_scal(in, out, size, elem_size); #endif return count; } /* ---- Wrappers for implementing blocking ---- */ /* Wrap a function for processing a single block to process an entire buffer in * parallel. */ int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, \ const size_t size, const size_t elem_size, size_t block_size) { omp_size_t ii = 0; int64_t err = 0; int64_t count, cum_count=0; size_t last_block_size; size_t leftover_bytes; size_t this_iter; char *last_in; char *last_out; ioc_chain C; ioc_init(&C, in, out); if (block_size == 0) { block_size = bshuf_default_block_size(elem_size); } if (block_size % BSHUF_BLOCKED_MULT) return -81; #if defined(_OPENMP) #pragma omp parallel for schedule(dynamic, 1) \ private(count) reduction(+ : cum_count) #endif for (ii = 0; ii < (omp_size_t)( size / block_size ); ii ++) { count = fun(&C, block_size, elem_size); if (count < 0) err = count; cum_count += count; } last_block_size = size % block_size; last_block_size = last_block_size - last_block_size % BSHUF_BLOCKED_MULT; if (last_block_size) { count = fun(&C, last_block_size, elem_size); if (count < 0) err = count; cum_count += count; } if (err < 0) return err; leftover_bytes = size % BSHUF_BLOCKED_MULT * elem_size; //this_iter; last_in = (char *) ioc_get_in(&C, &this_iter); ioc_set_next_in(&C, &this_iter, (void *) (last_in + leftover_bytes)); last_out = (char *) ioc_get_out(&C, &this_iter); ioc_set_next_out(&C, &this_iter, (void *) (last_out + leftover_bytes)); memcpy(last_out, last_in, leftover_bytes); ioc_destroy(&C); return cum_count + leftover_bytes; } /* Bitshuffle a single block. */ int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \ const size_t size, const size_t elem_size) { size_t this_iter; const void *in; void *out; int64_t count; in = ioc_get_in(C_ptr, &this_iter); ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size)); out = ioc_get_out(C_ptr, &this_iter); ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + size * elem_size)); count = bshuf_trans_bit_elem(in, out, size, elem_size); return count; } /* Bitunshuffle a single block. */ int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr, \ const size_t size, const size_t elem_size) { size_t this_iter; const void *in; void *out; int64_t count; in = ioc_get_in(C_ptr, &this_iter); ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size)); out = ioc_get_out(C_ptr, &this_iter); ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + size * elem_size)); count = bshuf_untrans_bit_elem(in, out, size, elem_size); return count; } /* Write a 64 bit unsigned integer to a buffer in big endian order. */ void bshuf_write_uint64_BE(void* buf, uint64_t num) { int ii; uint8_t* b = (uint8_t*) buf; uint64_t pow28 = 1 << 8; for (ii = 7; ii >= 0; ii--) { b[ii] = num % pow28; num = num / pow28; } } /* Read a 64 bit unsigned integer from a buffer big endian order. */ uint64_t bshuf_read_uint64_BE(void* buf) { int ii; uint8_t* b = (uint8_t*) buf; uint64_t num = 0, pow28 = 1 << 8, cp = 1; for (ii = 7; ii >= 0; ii--) { num += b[ii] * cp; cp *= pow28; } return num; } /* Write a 32 bit unsigned integer to a buffer in big endian order. */ void bshuf_write_uint32_BE(void* buf, uint32_t num) { int ii; uint8_t* b = (uint8_t*) buf; uint32_t pow28 = 1 << 8; for (ii = 3; ii >= 0; ii--) { b[ii] = num % pow28; num = num / pow28; } } /* Read a 32 bit unsigned integer from a buffer big endian order. */ uint32_t bshuf_read_uint32_BE(const void* buf) { int ii; uint8_t* b = (uint8_t*) buf; uint32_t num = 0, pow28 = 1 << 8, cp = 1; for (ii = 3; ii >= 0; ii--) { num += b[ii] * cp; cp *= pow28; } return num; } /* ---- Public functions ---- * * See header file for description and usage. * */ size_t bshuf_default_block_size(const size_t elem_size) { // This function needs to be absolutely stable between versions. // Otherwise encoded data will not be decodable. size_t block_size = BSHUF_TARGET_BLOCK_SIZE_B / elem_size; // Ensure it is a required multiple. block_size = (block_size / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT; return MAX(block_size, BSHUF_MIN_RECOMMEND_BLOCK); } int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size, const size_t elem_size, size_t block_size) { return bshuf_blocked_wrap_fun(&bshuf_bitshuffle_block, in, out, size, elem_size, block_size); } int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size, const size_t elem_size, size_t block_size) { return bshuf_blocked_wrap_fun(&bshuf_bitunshuffle_block, in, out, size, elem_size, block_size); } #undef TRANS_BIT_8X8 #undef TRANS_ELEM_TYPE #undef MAX #undef CHECK_MULT_EIGHT #undef CHECK_ERR_FREE #undef USESSE2 #undef USEAVX2 genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/bitshuffle/bitshuffle_core.h000066400000000000000000000100071453617025200304300ustar00rootroot00000000000000/* * Bitshuffle - Filter for improving compression of typed binary data. * * This file is part of Bitshuffle * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * See LICENSE file for details about copyright and rights to use. * * * Header File * * Worker routines return an int64_t which is the number of bytes processed * if positive or an error code if negative. * * Error codes: * -1 : Failed to allocate memory. * -11 : Missing SSE. * -12 : Missing AVX. * -13 : Missing Arm Neon. * -80 : Input size not a multiple of 8. * -81 : block_size not multiple of 8. * -91 : Decompression error, wrong number of bytes processed. * -1YYY : Error internal to compression routine with error code -YYY. */ #ifndef BITSHUFFLE_CORE_H #define BITSHUFFLE_CORE_H // We assume GNU g++ defining `__cplusplus` has stdint.h #if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus) #include #else typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; typedef signed int int32_t; typedef unsigned long long uint64_t; typedef long long int64_t; #endif #include // These are usually set in the setup.py. #ifndef BSHUF_VERSION_MAJOR #define BSHUF_VERSION_MAJOR 0 #define BSHUF_VERSION_MINOR 3 #define BSHUF_VERSION_POINT 5 #endif #ifdef __cplusplus extern "C" { #endif /* --- bshuf_using_SSE2 ---- * * Whether routines where compiled with the SSE2 instruction set. * * Returns * ------- * 1 if using SSE2, 0 otherwise. * */ int bshuf_using_SSE2(void); /* ---- bshuf_using_AVX2 ---- * * Whether routines where compiled with the AVX2 instruction set. * * Returns * ------- * 1 if using AVX2, 0 otherwise. * */ int bshuf_using_AVX2(void); /* ---- bshuf_default_block_size ---- * * The default block size as function of element size. * * This is the block size used by the blocked routines (any routine * taking a *block_size* argument) when the block_size is not provided * (zero is passed). * * The results of this routine are guaranteed to be stable such that * shuffled/compressed data can always be decompressed. * * Parameters * ---------- * elem_size : element size of data to be shuffled/compressed. * */ size_t bshuf_default_block_size(const size_t elem_size); /* ---- bshuf_bitshuffle ---- * * Bitshuffle the data. * * Transpose the bits within elements, in blocks of *block_size* * elements. * * Parameters * ---------- * in : input buffer, must be of size * elem_size bytes * out : output buffer, must be of size * elem_size bytes * size : number of elements in input * elem_size : element size of typed data * block_size : Do transpose in blocks of this many elements. Pass 0 to * select automatically (recommended). * * Returns * ------- * number of bytes processed, negative error-code if failed. * */ int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size, const size_t elem_size, size_t block_size); /* ---- bshuf_bitunshuffle ---- * * Unshuffle bitshuffled data. * * Untranspose the bits within elements, in blocks of *block_size* * elements. * * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size* * must match the parameters used to shuffle the data. * * Parameters * ---------- * in : input buffer, must be of size * elem_size bytes * out : output buffer, must be of size * elem_size bytes * size : number of elements in input * elem_size : element size of typed data * block_size : Do transpose in blocks of this many elements. Pass 0 to * select automatically (recommended). * * Returns * ------- * number of bytes processed, negative error-code if failed. * */ int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size, const size_t elem_size, size_t block_size); #ifdef __cplusplus } // extern "C" #endif #endif // BITSHUFFLE_CORE_H genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/bitshuffle/bitshuffle_internals.h000066400000000000000000000042251453617025200315040ustar00rootroot00000000000000/* * Bitshuffle - Filter for improving compression of typed binary data. * * This file is part of Bitshuffle * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * See LICENSE file for details about copyright and rights to use. */ #ifndef BITSHUFFLE_INTERNALS_H #define BITSHUFFLE_INTERNALS_H // We assume GNU g++ defining `__cplusplus` has stdint.h #if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus) #include #else typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; typedef signed int int32_t; typedef unsigned long long uint64_t; typedef long long int64_t; #endif #include #include "iochain.h" // Constants. #ifndef BSHUF_MIN_RECOMMEND_BLOCK #define BSHUF_MIN_RECOMMEND_BLOCK 128 #define BSHUF_BLOCKED_MULT 8 // Block sizes must be multiple of this. #define BSHUF_TARGET_BLOCK_SIZE_B 8192 #endif // Macros. #define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; } #ifdef __cplusplus extern "C" { #endif /* ---- Utility functions for internal use only ---- */ int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size, const size_t elem_size); /* Read a 32 bit unsigned integer from a buffer big endian order. */ uint32_t bshuf_read_uint32_BE(const void* buf); /* Write a 32 bit unsigned integer to a buffer in big endian order. */ void bshuf_write_uint32_BE(void* buf, uint32_t num); int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size, const size_t elem_size); /* Function definition for worker functions that process a single block. */ typedef int64_t (*bshufBlockFunDef)(ioc_chain* C_ptr, const size_t size, const size_t elem_size); /* Wrap a function for processing a single block to process an entire buffer in * parallel. */ int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, const size_t size, const size_t elem_size, size_t block_size); #ifdef __cplusplus } // extern "C" #endif #endif // BITSHUFFLE_INTERNALS_H genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/bitshuffle/iochain.c000066400000000000000000000046641453617025200267060ustar00rootroot00000000000000/* * IOchain - Distribute a chain of dependant IO events amoung threads. * * This file is part of Bitshuffle * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * See LICENSE file for details about copyright and rights to use. * */ #include #include "iochain.h" void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0) { #ifdef _OPENMP omp_init_lock(&C->next_lock); for (size_t ii = 0; ii < IOC_SIZE; ii ++) { omp_init_lock(&(C->in_pl[ii].lock)); omp_init_lock(&(C->out_pl[ii].lock)); } #endif C->next = 0; C->in_pl[0].ptr = in_ptr_0; C->out_pl[0].ptr = out_ptr_0; } void ioc_destroy(ioc_chain *C) { #ifdef _OPENMP omp_destroy_lock(&C->next_lock); for (size_t ii = 0; ii < IOC_SIZE; ii ++) { omp_destroy_lock(&(C->in_pl[ii].lock)); omp_destroy_lock(&(C->out_pl[ii].lock)); } #endif } const void * ioc_get_in(ioc_chain *C, size_t *this_iter) { #ifdef _OPENMP omp_set_lock(&C->next_lock); #pragma omp flush #endif *this_iter = C->next; C->next ++; #ifdef _OPENMP omp_set_lock(&(C->in_pl[*this_iter % IOC_SIZE].lock)); omp_set_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock)); omp_set_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock)); omp_unset_lock(&C->next_lock); #endif return C->in_pl[*this_iter % IOC_SIZE].ptr; } void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr) { C->in_pl[(*this_iter + 1) % IOC_SIZE].ptr = in_ptr; #ifdef _OPENMP omp_unset_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock)); #endif } void * ioc_get_out(ioc_chain *C, size_t *this_iter) { #ifdef _OPENMP omp_set_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock)); #pragma omp flush #endif void *out_ptr = C->out_pl[*this_iter % IOC_SIZE].ptr; #ifdef _OPENMP omp_unset_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock)); #endif return out_ptr; } void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) { C->out_pl[(*this_iter + 1) % IOC_SIZE].ptr = out_ptr; #ifdef _OPENMP omp_unset_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock)); // *in_pl[this_iter]* lock released at the end of the iteration to avoid being // overtaken by previous threads and having *out_pl[this_iter]* corrupted. // Especially worried about thread 0, iteration 0. omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock)); #endif } genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/bitshuffle/iochain.h000066400000000000000000000050711453617025200267040ustar00rootroot00000000000000/* * IOchain - Distribute a chain of dependant IO events amoung threads. * * This file is part of Bitshuffle * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * See LICENSE file for details about copyright and rights to use. * * * Header File * * Similar in concept to a queue. Each task includes reading an input * and writing output, but the location of the input/output (the pointers) * depend on the previous item in the chain. * * This is designed for parallelizing blocked compression/decompression IO, * where the destination of a compressed block depends on the compressed size * of all previous blocks. * * Implemented with OpenMP locks. * * * Usage * ----- * - Call `ioc_init` in serial block. * - Each thread should create a local variable *size_t this_iter* and * pass its address to all function calls. Its value will be set * inside the functions and is used to identify the thread. * - Each thread must call each of the `ioc_get*` and `ioc_set*` methods * exactly once per iteration, starting with `ioc_get_in` and ending * with `ioc_set_next_out`. * - The order (`ioc_get_in`, `ioc_set_next_in`, *work*, `ioc_get_out`, * `ioc_set_next_out`, *work*) is most efficient. * - Have each thread call `ioc_end_pop`. * - `ioc_get_in` is blocked until the previous entry's * `ioc_set_next_in` is called. * - `ioc_get_out` is blocked until the previous entry's * `ioc_set_next_out` is called. * - There are no blocks on the very first iteration. * - Call `ioc_destroy` in serial block. * - Safe for num_threads >= IOC_SIZE (but less efficient). * */ #ifndef IOCHAIN_H #define IOCHAIN_H #include #ifdef _OPENMP #include #endif #define IOC_SIZE 33 typedef struct ioc_ptr_and_lock { #ifdef _OPENMP omp_lock_t lock; #endif void *ptr; } ptr_and_lock; typedef struct ioc_const_ptr_and_lock { #ifdef _OPENMP omp_lock_t lock; #endif const void *ptr; } const_ptr_and_lock; typedef struct ioc_chain { #ifdef _OPENMP omp_lock_t next_lock; #endif size_t next; const_ptr_and_lock in_pl[IOC_SIZE]; ptr_and_lock out_pl[IOC_SIZE]; } ioc_chain; void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0); void ioc_destroy(ioc_chain *C); const void * ioc_get_in(ioc_chain *C, size_t *this_iter); void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr); void * ioc_get_out(ioc_chain *C, size_t *this_iter); void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr); #endif // IOCHAIN_H genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/lz4/000077500000000000000000000000001453617025200234745ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/lz4/LICENSE000066400000000000000000000024371453617025200245070ustar00rootroot00000000000000LZ4 Library Copyright (c) 2011-2016, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/lz4/README.md000066400000000000000000000120331453617025200247520ustar00rootroot00000000000000LZ4 - Library Files ================================ The `/lib` directory contains many files, but depending on project's objectives, not all of them are necessary. #### Minimal LZ4 build The minimum required is **`lz4.c`** and **`lz4.h`**, which provides the fast compression and decompression algorithms. They generate and decode data using the [LZ4 block format]. #### High Compression variant For more compression ratio at the cost of compression speed, the High Compression variant called **lz4hc** is available. Add files **`lz4hc.c`** and **`lz4hc.h`**. This variant also compresses data using the [LZ4 block format], and depends on regular `lib/lz4.*` source files. #### Frame support, for interoperability In order to produce compressed data compatible with `lz4` command line utility, it's necessary to use the [official interoperable frame format]. This format is generated and decoded automatically by the **lz4frame** library. Its public API is described in `lib/lz4frame.h`. In order to work properly, lz4frame needs all other modules present in `/lib`, including, lz4 and lz4hc, and also **xxhash**. So it's necessary to include all `*.c` and `*.h` files present in `/lib`. #### Advanced / Experimental API Definitions which are not guaranteed to remain stable in future versions, are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`. As the name implies, these definitions can only be invoked in the context of static linking ***only***. Otherwise, dependent application may fail on API or ABI break in the future. The associated symbols are also not present in dynamic library by default. Should they be nonetheless needed, it's possible to force their publication by using build macro `LZ4_PUBLISH_STATIC_FUNCTIONS`. #### Build macros The following build macro can be selected at compilation time : - `LZ4_FAST_DEC_LOOP` : this triggers the optimized decompression loop. This loops works great on x86/x64 cpus, and is automatically enabled on this platform. It's possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor. For example, with `gcc` : `-DLZ4_FAST_DEC_LOOP=1`, and with `make` : `CPPFLAGS+=-DLZ4_FAST_DEC_LOOP=1 make lz4`. - `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow. Set to 65535 by default, which is the maximum value supported by lz4 format. Reducing maximum distance will reduce opportunities for LZ4 to find matches, hence will produce a worse compression ratio. However, a smaller max distance can allow compatibility with specific decoders using limited memory budget. This build macro only influences the compressed output of the compressor. - `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning. This is meant to invite users to update their source code. Should this be a problem, it's generally possible to make the compiler ignore these warnings, for example with `-Wno-deprecated-declarations` on `gcc`, or `_CRT_SECURE_NO_WARNINGS` for Visual Studio. Another method is to define `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files. #### Amalgamation lz4 source code can be amalgamated into a single file. One can combine all source code into `lz4_all.c` by using following command: ``` cat lz4.c lz4hc.c lz4frame.c > lz4_all.c ``` (`cat` file order is important) then compile `lz4_all.c`. All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`. #### Windows : using MinGW+MSYS to create DLL DLL can be created using MinGW+MSYS with the `make liblz4` command. This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits: ``` make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT ``` The import library is only required with Visual C++. The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` file it should be linked with `dll\liblz4.dll`. For example: ``` $(CC) $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. #### Miscellaneous Other files present in the directory are not source code. There are : - `LICENSE` : contains the BSD license text - `Makefile` : `make` script to compile and install lz4 library (static and dynamic) - `liblz4.pc.in` : for `pkg-config` (used in `make install`) - `README.md` : this file [official interoperable frame format]: ../doc/lz4_Frame_format.md [LZ4 block format]: ../doc/lz4_Block_format.md #### License All source material within __lib__ directory are BSD 2-Clause licensed. See [LICENSE](LICENSE) for details. The license is also reminded at the top of each source file. genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/lz4/lz4.c000066400000000000000000003056441453617025200243650ustar00rootroot00000000000000/* LZ4 - Fast LZ compression algorithm Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - LZ4 homepage : http://www.lz4.org - LZ4 source repository : https://github.com/lz4/lz4 */ /*-************************************ * Tuning parameters **************************************/ /* * LZ4_HEAPMODE : * Select how default compression functions will allocate memory for their hash table, * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ #ifndef LZ4_HEAPMODE # define LZ4_HEAPMODE 0 #endif /* * ACCELERATION_DEFAULT : * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 */ #define ACCELERATION_DEFAULT 1 /*-************************************ * CPU Feature Detection **************************************/ /* LZ4_FORCE_MEMORY_ACCESS * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. * The below switch allow to select different access method for improved performance. * Method 0 (default) : use `memcpy()`. Safe and portable. * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets which assembly generation depends on alignment. * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ # if defined(__GNUC__) && \ ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define LZ4_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) # define LZ4_FORCE_MEMORY_ACCESS 1 # endif #endif /* * LZ4_FORCE_SW_BITCOUNT * Define this parameter if your target system or compiler does not support hardware bit count */ #if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ # define LZ4_FORCE_SW_BITCOUNT #endif /*-************************************ * Dependency **************************************/ /* * LZ4_SRC_INCLUDED: * Amalgamation flag, whether lz4.c is included */ #ifndef LZ4_SRC_INCLUDED # define LZ4_SRC_INCLUDED 1 #endif #ifndef LZ4_STATIC_LINKING_ONLY #define LZ4_STATIC_LINKING_ONLY #endif #ifndef LZ4_DISABLE_DEPRECATE_WARNINGS #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ #endif #define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* see also "memory routines" below */ /*-************************************ * Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ # include # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #endif /* _MSC_VER */ #ifndef LZ4_FORCE_INLINE # ifdef _MSC_VER /* Visual Studio */ # define LZ4_FORCE_INLINE static __forceinline # else # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ # define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) # else # define LZ4_FORCE_INLINE static inline # endif # else # define LZ4_FORCE_INLINE static # endif /* __STDC_VERSION__ */ # endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ /* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, * together with a simple 8-byte copy loop as a fall-back path. * However, this optimization hurts the decompression speed by >30%, * because the execution does not go to the optimized loop * for typical compressible data, and all of the preamble checks * before going to the fall-back path become useless overhead. * This optimization happens only with the -O3 flag, and -O2 generates * a simple 8-byte copy loop. * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 * functions are annotated with __attribute__((optimize("O2"))), * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute * of LZ4_wildCopy8 does not affect the compression speed. */ #if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) # define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE #else # define LZ4_FORCE_O2_GCC_PPC64LE # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static #endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) #endif #ifndef likely #define likely(expr) expect((expr) != 0, 1) #endif #ifndef unlikely #define unlikely(expr) expect((expr) != 0, 0) #endif /*-************************************ * Memory routines **************************************/ #include /* malloc, calloc, free */ #define ALLOC(s) malloc(s) #define ALLOC_AND_ZERO(s) calloc(1,s) #define FREEMEM(p) free(p) #include /* memset, memcpy */ #define MEM_INIT(p,v,s) memset((p),(v),(s)) /*-************************************ * Common Constants **************************************/ #define MINMATCH 4 #define WILDCOPYLENGTH 8 #define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ #define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ #define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ #define FASTLOOP_SAFE_DISTANCE 64 static const int LZ4_minLength = (MFLIMIT+1); #define KB *(1 <<10) #define MB *(1 <<20) #define GB *(1U<<30) #define LZ4_DISTANCE_ABSOLUTE_MAX 65535 #if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ # error "LZ4_DISTANCE_MAX is too big : must be <= 65535" #endif #define ML_BITS 4 #define ML_MASK ((1U<=1) # include #else # ifndef assert # define assert(condition) ((void)0) # endif #endif #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) # include static int g_debuglog_enable = 1; # define DEBUGLOG(l, ...) { \ if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ fprintf(stderr, __FILE__ ": "); \ fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, " \n"); \ } } #else # define DEBUGLOG(l, ...) {} /* disabled */ #endif /*-************************************ * Types **************************************/ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; typedef uint64_t U64; typedef uintptr_t uptrval; #else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; typedef size_t uptrval; /* generally true, except OpenVMS-64 */ #endif #if defined(__x86_64__) typedef U64 reg_t; /* 64-bits in x32 mode */ #else typedef size_t reg_t; /* 32-bits in x32 mode */ #endif typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive; /*-************************************ * Reading and writing into memory **************************************/ static unsigned LZ4_isLittleEndian(void) { const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; } #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) /* lie to the compiler about data alignment; use with caution */ static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } #elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } #else /* safe and portable access using memcpy() */ static U16 LZ4_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; } static U32 LZ4_read32(const void* memPtr) { U32 val; memcpy(&val, memPtr, sizeof(val)); return val; } static reg_t LZ4_read_ARCH(const void* memPtr) { reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; } static void LZ4_write16(void* memPtr, U16 value) { memcpy(memPtr, &value, sizeof(value)); } static void LZ4_write32(void* memPtr, U32 value) { memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ static U16 LZ4_readLE16(const void* memPtr) { if (LZ4_isLittleEndian()) { return LZ4_read16(memPtr); } else { const BYTE* p = (const BYTE*)memPtr; return (U16)((U16)p[0] + (p[1]<<8)); } } static void LZ4_writeLE16(void* memPtr, U16 value) { if (LZ4_isLittleEndian()) { LZ4_write16(memPtr, value); } else { BYTE* p = (BYTE*)memPtr; p[0] = (BYTE) value; p[1] = (BYTE)(value>>8); } } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; do { memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE void LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH * - there is at least 8 bytes available to write after dstEnd */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE void LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { BYTE v[8]; assert(dstEnd >= dstPtr + MINMATCH); LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */ switch(offset) { case 1: memset(v, *srcPtr, 8); break; case 2: memcpy(v, srcPtr, 2); memcpy(&v[2], srcPtr, 2); memcpy(&v[4], &v[0], 4); break; case 4: memcpy(v, srcPtr, 4); memcpy(&v[4], srcPtr, 4); break; default: LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); return; } memcpy(dstPtr, v, 8); dstPtr += 8; while (dstPtr < dstEnd) { memcpy(dstPtr, v, 8); dstPtr += 8; } } #endif /*-************************************ * Common functions **************************************/ static unsigned LZ4_NbCommonBytes (reg_t val) { if (LZ4_isLittleEndian()) { if (sizeof(val)==8) { # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_ctzll((U64)val) >> 3; # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif } else /* 32 bits */ { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r; _BitScanForward( &r, (U32)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_ctz((U32)val) >> 3; # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif } } else /* Big Endian CPU */ { if (sizeof(val)==8) { /* 64-bits */ # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_clzll((U64)val) >> 3; # else static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. Note that this code path is never triggered in 32-bits mode. */ unsigned r; if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; # endif } else /* 32 bits */ { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (unsigned)__builtin_clz((U32)val) >> 3; # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } r += (!val); return r; # endif } } } #define STEPSIZE sizeof(reg_t) LZ4_FORCE_INLINE unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; if (likely(pIn < pInLimit-(STEPSIZE-1))) { reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; } else { return LZ4_NbCommonBytes(diff); } } while (likely(pIn < pInLimit-(STEPSIZE-1))) { reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } pIn += LZ4_NbCommonBytes(diff); return (unsigned)(pIn - pStart); } if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } if ((pIn compression run slower on incompressible data */ /*-************************************ * Local Structures and types **************************************/ typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; /** * This enum distinguishes several different modes of accessing previous * content in the stream. * * - noDict : There is no preceding content. * - withPrefix64k : Table entries up to ctx->dictSize before the current blob * blob being compressed are valid and refer to the preceding * content (of length ctx->dictSize), which is available * contiguously preceding in memory the content currently * being compressed. * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere * else in memory, starting at ctx->dictionary with length * ctx->dictSize. * - usingDictCtx : Like usingExtDict, but everything concerning the preceding * content is in a separate context, pointed to by * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table * entries in the current context that refer to positions * preceding the beginning of the current compression are * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx * ->dictSize describe the location and size of the preceding * content, and matches are found by looking in the ctx * ->dictCtx->hashTable. */ typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; /*-************************************ * Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } /*-************************************ * Internal Definitions used in Tests **************************************/ #if defined (__cplusplus) extern "C" { #endif int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const void* dictStart, size_t dictSize); #if defined (__cplusplus) } #endif /*-****************************** * Compression functions ********************************/ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); else return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; if (LZ4_isLittleEndian()) { const U64 prime5bytes = 889523592379ULL; return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); } else { const U64 prime8bytes = 11400714785074694791ULL; return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } } LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) { if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); return LZ4_hash4(LZ4_read32(p), tableType); } static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) { default: /* fallthrough */ case clearedTable: { /* illegal! */ assert(0); return; } case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } } } static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) { default: /* fallthrough */ case clearedTable: /* fallthrough */ case byPtr: { /* illegal! */ assert(0); return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } } } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { switch (tableType) { case clearedTable: { /* illegal! */ assert(0); return; } case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } } } LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } /* LZ4_getIndexOnHash() : * Index of match position registered in hash table. * hash position must be calculated by using base+index, or dictBase+index. * Assumption 1 : only valid if tableType == byU32 or byU16. * Assumption 2 : h is presumed valid (within limits of hash table) */ static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) { LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; } if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; } assert(0); return 0; /* forbidden case */ } static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } LZ4_FORCE_INLINE void LZ4_prepareTable(LZ4_stream_t_internal* const cctx, const int inputSize, const tableType_t tableType) { /* If compression failed during the previous step, then the context * is marked as dirty, therefore, it has to be fully reset. */ if (cctx->dirty) { DEBUGLOG(5, "LZ4_prepareTable: Full reset for %p", cctx); MEM_INIT(cctx, 0, sizeof(LZ4_stream_t_internal)); return; } /* If the table hasn't been used, it's guaranteed to be zeroed out, and is * therefore safe to use no matter what mode we're in. Otherwise, we figure * out if it's safe to leave as is or whether it needs to be reset. */ if (cctx->tableType != clearedTable) { assert(inputSize >= 0); if (cctx->tableType != tableType || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) || ((tableType == byU32) && cctx->currentOffset > 1 GB) || tableType == byPtr || inputSize >= 4 KB) { DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); cctx->currentOffset = 0; cctx->tableType = clearedTable; } else { DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); } } /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster * than compressing without a gap. However, compressing with * currentOffset == 0 is faster still, so we preserve that case. */ if (cctx->currentOffset != 0 && tableType == byU32) { DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); cctx->currentOffset += 64 KB; } /* Finally, clear history */ cctx->dictCtx = NULL; cctx->dictionary = NULL; cctx->dictSize = 0; } /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ LZ4_FORCE_INLINE int LZ4_compress_generic( LZ4_stream_t_internal* const cctx, const char* const source, char* const dest, const int inputSize, int *inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, const limitedOutput_directive outputDirective, const tableType_t tableType, const dict_directive dictDirective, const dictIssue_directive dictIssue, const int acceleration) { int result; const BYTE* ip = (const BYTE*) source; U32 const startIndex = cctx->currentOffset; const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; const BYTE* const dictionary = dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; const U32 dictSize = dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ const BYTE* const dictEnd = dictionary + dictSize; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; const BYTE* const matchlimit = iend - LASTLITERALS; /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ const BYTE* dictBase = (dictDirective == usingDictCtx) ? dictionary + dictSize - dictCtx->currentOffset : dictionary + dictSize - startIndex; BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; U32 offset = 0; U32 forwardH; DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); /* If init conditions are not met, we don't have to mark stream * as having dirty context, since no action was taken yet */ if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); /* Update context state */ if (dictDirective == usingDictCtx) { /* Subsequent linked blocks can't use the dictionary. */ /* Instead, they use the block we just compressed. */ cctx->dictCtx = NULL; cctx->dictSize = (U32)inputSize; } else { cctx->dictSize += (U32)inputSize; } cctx->currentOffset += (U32)inputSize; cctx->tableType = (U16)tableType; if (inputSizehashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ for ( ; ; ) { const BYTE* match; BYTE* token; const BYTE* filledIp; /* Find a match */ if (tableType == byPtr) { const BYTE* forwardIp = ip; int step = 1; int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; ip = forwardIp; forwardIp += step; step = (searchMatchNb++ >> LZ4_skipTrigger); if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; assert(ip < mflimitPlusOne); match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); } while ( (match+LZ4_DISTANCE_MAX < ip) || (LZ4_read32(match) != LZ4_read32(ip)) ); } else { /* byU32, byU16 */ const BYTE* forwardIp = ip; int step = 1; int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; U32 const current = (U32)(forwardIp - base); U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); assert(matchIndex <= current); assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); ip = forwardIp; forwardIp += step; step = (searchMatchNb++ >> LZ4_skipTrigger); if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; assert(ip < mflimitPlusOne); if (dictDirective == usingDictCtx) { if (matchIndex < startIndex) { /* there was no match, try the dictionary */ assert(tableType == byU32); matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); match = dictBase + matchIndex; matchIndex += dictDelta; /* make dictCtx index comparable with current context */ lowLimit = dictionary; } else { match = base + matchIndex; lowLimit = (const BYTE*)source; } } else if (dictDirective==usingExtDict) { if (matchIndex < startIndex) { DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); assert(startIndex - matchIndex >= MINMATCH); match = dictBase + matchIndex; lowLimit = dictionary; } else { match = base + matchIndex; lowLimit = (const BYTE*)source; } } else { /* single continuous memory segment */ match = base + matchIndex; } forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ assert(matchIndex < current); if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) && (matchIndex+LZ4_DISTANCE_MAX < current)) { continue; } /* too far */ assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ if (LZ4_read32(match) == LZ4_read32(ip)) { if (maybe_extMem) offset = current - matchIndex; break; /* match found */ } } while(1); } /* Catch up */ filledIp = ip; while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ } if ((outputDirective == fillOutput) && (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { op--; goto _last_literals; } if (litLength >= RUN_MASK) { int len = (int)(litLength - RUN_MASK); *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } else *token = (BYTE)(litLength< olimit)) { /* the match was too close to the end, rewind and go to last literals */ op = token; goto _last_literals; } /* Encode Offset */ if (maybe_extMem) { /* static test */ DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); assert(offset <= LZ4_DISTANCE_MAX && offset > 0); LZ4_writeLE16(op, (U16)offset); op+=2; } else { DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); assert(ip-match <= LZ4_DISTANCE_MAX); LZ4_writeLE16(op, (U16)(ip - match)); op+=2; } /* Encode MatchLength */ { unsigned matchCode; if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) && (lowLimit==dictionary) /* match within extDict */ ) { const BYTE* limit = ip + (dictEnd-match); assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += (size_t)matchCode + MINMATCH; if (ip==limit) { unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; ip += more; } DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += (size_t)matchCode + MINMATCH; DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } if ((outputDirective) && /* Check output buffer overflow */ (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { if (outputDirective == fillOutput) { /* Match description too long : reduce it */ U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; ip -= matchCode - newMatchCode; assert(newMatchCode < matchCode); matchCode = newMatchCode; if (unlikely(ip <= filledIp)) { /* We have already filled up to filledIp so if ip ends up less than filledIp * we have positions in the hash table beyond the current position. This is * a problem if we reuse the hash table. So we have to remove these positions * from the hash table. */ const BYTE* ptr; DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); for (ptr = ip; ptr <= filledIp; ++ptr) { U32 const h = LZ4_hashPosition(ptr, tableType); LZ4_clearHash(h, cctx->hashTable, tableType); } } } else { assert(outputDirective == limitedOutput); return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ } } if (matchCode >= ML_MASK) { *token += ML_MASK; matchCode -= ML_MASK; LZ4_write32(op, 0xFFFFFFFF); while (matchCode >= 4*255) { op+=4; LZ4_write32(op, 0xFFFFFFFF); matchCode -= 4*255; } op += matchCode / 255; *op++ = (BYTE)(matchCode % 255); } else *token += (BYTE)(matchCode); } /* Ensure we have enough space for the last literals. */ assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); anchor = ip; /* Test end of chunk */ if (ip >= mflimitPlusOne) break; /* Fill table */ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); /* Test next position */ if (tableType == byPtr) { match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); LZ4_putPosition(ip, cctx->hashTable, tableType, base); if ( (match+LZ4_DISTANCE_MAX >= ip) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } } else { /* byU32, byU16 */ U32 const h = LZ4_hashPosition(ip, tableType); U32 const current = (U32)(ip-base); U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); assert(matchIndex < current); if (dictDirective == usingDictCtx) { if (matchIndex < startIndex) { /* there was no match, try the dictionary */ matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); match = dictBase + matchIndex; lowLimit = dictionary; /* required for match length counter */ matchIndex += dictDelta; } else { match = base + matchIndex; lowLimit = (const BYTE*)source; /* required for match length counter */ } } else if (dictDirective==usingExtDict) { if (matchIndex < startIndex) { match = dictBase + matchIndex; lowLimit = dictionary; /* required for match length counter */ } else { match = base + matchIndex; lowLimit = (const BYTE*)source; /* required for match length counter */ } } else { /* single memory segment */ match = base + matchIndex; } LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); assert(matchIndex < current); if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; if (maybe_extMem) offset = current - matchIndex; DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); goto _next_match; } } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); } _last_literals: /* Encode Last Literals */ { size_t lastRun = (size_t)(iend - anchor); if ( (outputDirective) && /* Check output buffer overflow */ (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { if (outputDirective == fillOutput) { /* adapt lastRun to fill 'dst' */ assert(olimit >= op); lastRun = (size_t)(olimit-op) - 1; lastRun -= (lastRun+240)/255; } else { assert(outputDirective == limitedOutput); return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ } } if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; *op++ = (BYTE) accumulator; } else { *op++ = (BYTE)(lastRun< 0); return result; } int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; assert(ctx != NULL); if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); } else { const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } } /** * LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). * * Using this variant avoids an expensive initialization step. It is only safe * to call if the state buffer is known to be correctly initialized already * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of * "correctly initialized"). */ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (dstCapacity >= LZ4_compressBound(srcSize)) { if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; LZ4_prepareTable(ctx, srcSize, tableType); if (ctx->currentOffset) { return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); } else { return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; LZ4_prepareTable(ctx, srcSize, tableType); return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { if (srcSize < LZ4_64Klimit) { const tableType_t tableType = byU16; LZ4_prepareTable(ctx, srcSize, tableType); if (ctx->currentOffset) { return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); } else { return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } else { const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; LZ4_prepareTable(ctx, srcSize, tableType); return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } } } int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { int result; #if (LZ4_HEAPMODE) LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctxPtr == NULL) return 0; #else LZ4_stream_t ctx; LZ4_stream_t* const ctxPtr = &ctx; #endif result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (LZ4_HEAPMODE) FREEMEM(ctxPtr); #endif return result; } int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize) { return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1); } /* hidden debug function */ /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { LZ4_stream_t ctx; LZ4_initStream(&ctx, sizeof(ctx)); if (srcSize < LZ4_64Klimit) { return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { tableType_t const addrMode = (sizeof(void*) > 4) ? byU32 : byPtr; return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, addrMode, noDict, noDictIssue, acceleration); } } /* Note!: This function leaves the stream in an unclean/broken state! * It is not safe to subsequently use the same state with a _fastReset() or * _continue() call without resetting it. */ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) { void* const s = LZ4_initStream(state, sizeof (*state)); assert(s != NULL); (void)s; if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { if (*srcSizePtr < LZ4_64Klimit) { return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); } else { tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); } } } int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (LZ4_HEAPMODE) LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctx == NULL) return 0; #else LZ4_stream_t ctxBody; LZ4_stream_t* ctx = &ctxBody; #endif int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); #if (LZ4_HEAPMODE) FREEMEM(ctx); #endif return result; } /*-****************************** * Streaming functions ********************************/ LZ4_stream_t* LZ4_createStream(void) { LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ DEBUGLOG(4, "LZ4_createStream %p", lz4s); if (lz4s == NULL) return NULL; LZ4_initStream(lz4s, sizeof(*lz4s)); return lz4s; } #ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : it reports an aligment of 8-bytes, while actually aligning LZ4_stream_t on 4 bytes. */ static size_t LZ4_stream_t_alignment(void) { struct { char c; LZ4_stream_t t; } t_a; return sizeof(t_a) - sizeof(t_a.t); } #endif LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) { DEBUGLOG(5, "LZ4_initStream"); if (buffer == NULL) { return NULL; } if (size < sizeof(LZ4_stream_t)) { return NULL; } #ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : it reports an aligment of 8-bytes, while actually aligning LZ4_stream_t on 4 bytes. */ if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ #endif MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); return (LZ4_stream_t*)buffer; } /* resetStream is now deprecated, * prefer initStream() which is more general */ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } void LZ4_resetStream_fast(LZ4_stream_t* ctx) { LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); FREEMEM(LZ4_stream); return (0); } #define HASH_UNIT sizeof(reg_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); /* It's necessary to reset the context, * and not just continue it with prepareTable() * to avoid any risk of generating overflowing matchIndex * when compressing using this dictionary */ LZ4_resetStream(LZ4_dict); /* We always increment the offset by 64 KB, since, if the dict is longer, * we truncate it to the last 64k, and if it's shorter, we still want to * advance by a whole window length so we can provide the guarantee that * there are only valid offsets in the window, which allows an optimization * in LZ4_compress_fast_continue() where it uses noDictIssue even when the * dictionary isn't a full 64k. */ dict->currentOffset += 64 KB; if (dictSize < (int)HASH_UNIT) { return 0; } if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); dict->tableType = tableType; while (p <= dictEnd-HASH_UNIT) { LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; } return (int)dict->dictSize; } void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : &(dictionaryStream->internal_donotuse); DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", workingStream, dictionaryStream, dictCtx != NULL ? dictCtx->dictSize : 0); /* Calling LZ4_resetStream_fast() here makes sure that changes will not be * erased by subsequent calls to LZ4_resetStream_fast() in case stream was * marked as having dirty context, e.g. requiring full reset. */ LZ4_resetStream_fast(workingStream); if (dictCtx != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table * entry can take that indicate a miss. In that case, we need * to bump the offset to something non-zero. */ if (workingStream->internal_donotuse.currentOffset == 0) { workingStream->internal_donotuse.currentOffset = 64 KB; } /* Don't actually attach an empty dictionary. */ if (dictCtx->dictSize == 0) { dictCtx = NULL; } } workingStream->internal_donotuse.dictCtx = dictCtx; } static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { assert(nextSize >= 0); if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; DEBUGLOG(4, "LZ4_renormDictT"); for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; } LZ4_dict->currentOffset = 64 KB; if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; } } int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { const tableType_t tableType = byU32; LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; /* invalidate tiny dictionaries */ if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ && (dictEnd != (const BYTE*)source) ) { DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); streamPtr->dictSize = 0; streamPtr->dictionary = (const BYTE*)source; dictEnd = (const BYTE*)source; } /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; streamPtr->dictionary = dictEnd - streamPtr->dictSize; } } /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); } /* external dictionary mode */ { int result; if (streamPtr->dictCtx) { /* We depend here on the fact that dictCtx'es (produced by * LZ4_loadDict) guarantee that their tables contain no references * to offsets between dictCtx->currentOffset - 64 KB and * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe * to use noDictIssue even when the dict isn't a full 64 KB. */ if (inputSize > 4 KB) { /* For compressing large blobs, it is faster to pay the setup * cost to copy the dictionary's tables into the active context, * so that the compression loop is only looking into one table. */ memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t)); result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } } else { if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); } } streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; return result; } } /* Hidden debug function, to force-test external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; LZ4_renormDictT(streamPtr, srcSize); if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); } else { result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); } streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)srcSize; return result; } /*! LZ4_saveDict() : * If previously compressed data block is not guaranteed to remain available at its memory location, * save it into a safer place (char* safeBuffer). * Note : you don't need to call LZ4_loadDict() afterwards, * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; return dictSize; } /*-******************************* * Decompression functions ********************************/ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; #undef MIN #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) /* Read the variable-length literal or match length. * * ip - pointer to use as input. * lencheck - end ip. Return an error if ip advances >= lencheck. * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. * error (output) - error code. Should be set to 0 before call. */ typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; LZ4_FORCE_INLINE unsigned read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error) { unsigned length = 0; unsigned s; if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ *error = initial_error; return length; } do { s = **ip; (*ip)++; length += s; if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ *error = loop_error; return length; } } while (s==255); return length; } /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ LZ4_FORCE_INLINE int LZ4_decompress_generic( const char* const src, char* const dst, int srcSize, int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */ earlyEnd_directive partialDecoding, /* full, partial */ dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ const BYTE* const dictStart, /* only if dict==usingExtDict */ const size_t dictSize /* note : = 0 if noDict */ ) { if (src == NULL) { return -1; } { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; BYTE* op = (BYTE*) dst; BYTE* const oend = op + outputSize; BYTE* cpy; const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; const int safeDecode = (endOnInput==endOnInputSize); const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); /* Set up the "end" pointers for the shortcut. */ const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; const BYTE* match; size_t offset; unsigned token; size_t length; DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); /* Special cases */ assert(lowPrefix <= op); if ((endOnInput) && (unlikely(outputSize==0))) { /* Empty output buffer */ if (partialDecoding) return 0; return ((srcSize==1) && (*ip==0)) ? 0 : -1; } if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); } if ((endOnInput) && unlikely(srcSize==0)) { return -1; } /* Currently the fast loop shows a regression on qualcomm arm chips. */ #if LZ4_FAST_DEC_LOOP if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { DEBUGLOG(6, "skip fast decode loop"); goto safe_decode; } /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ while (1) { /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); if (endOnInput) { assert(ip < iend); } token = *ip++; length = token >> ML_BITS; /* literal length */ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ /* decode literal length */ if (length == RUN_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); if (error == initial_error) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ /* copy literals */ cpy = op+length; LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); if (endOnInput) { /* LZ4_decompress_safe() */ if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } LZ4_wildCopy32(op, ip, cpy); } else { /* LZ4_decompress_fast() */ if (cpy>oend-8) { goto safe_literal_copy; } LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and only relies on end-of-block properties */ } ip += length; op = cpy; } else { cpy = op+length; if (endOnInput) { /* LZ4_decompress_safe() */ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); /* We don't need to check oend, since we check it once for each loop below */ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ memcpy(op, ip, 16); } else { /* LZ4_decompress_fast() */ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and relies on end-of-block properties */ memcpy(op, ip, 8); if (length > 8) { memcpy(op+8, ip+8, 8); } } ip += length; op = cpy; } /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; assert(match <= op); /* get matchlength */ length = token & ML_MASK; if (length == ML_MASK) { variable_length_error error = ok; if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); if (error != ok) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ length += MINMATCH; if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; } } else { length += MINMATCH; if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; } /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ if ((dict == withPrefix64k) || (match >= lowPrefix)) { if (offset >= 8) { assert(match >= lowPrefix); assert(match <= op); assert(op + 18 <= oend); memcpy(op, match, 8); memcpy(op+8, match+8, 8); memcpy(op+16, match+16, 2); op += length; continue; } } } if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { if (partialDecoding) { length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */ } else { goto _output_error; /* end-of-block condition violated */ } } if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ memmove(op, dictEnd - (lowPrefix-match), length); op += length; } else { /* match stretches into both external dictionary and current block */ size_t const copySize = (size_t)(lowPrefix - match); size_t const restSize = length - copySize; memcpy(op, dictEnd - copySize, copySize); op += copySize; if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) { *op++ = *copyFrom++; } } else { memcpy(op, lowPrefix, restSize); op += restSize; } } continue; } /* copy match within block */ cpy = op + length; assert((op <= oend) && (oend-op >= 32)); if (unlikely(offset<16)) { LZ4_memcpy_using_offset(op, match, cpy, offset); } else { LZ4_wildCopy32(op, match, cpy); } op = cpy; /* wildcopy correction */ } safe_decode: #endif /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ while (1) { token = *ip++; length = token >> ML_BITS; /* literal length */ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ /* A two-stage shortcut for the most common case: * 1) If the literal length is 0..14, and there is enough space, * enter the shortcut and copy 16 bytes on behalf of the literals * (in the fast mode, only 8 bytes can be safely copied this way). * 2) Further if the match length is 4..18, copy 18 bytes in a similar * manner; but we ensure that there's enough space in the output for * those 18 bytes earlier, upon entering the shortcut (in other words, * there is a combined check for both stages). */ if ( (endOnInput ? length != RUN_MASK : length <= 8) /* strictly "less than" on input, to re-enter the loop with at least one byte */ && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { /* Copy the literals */ memcpy(op, ip, endOnInput ? 16 : 8); op += length; ip += length; /* The second stage: prepare for match copying, decode full info. * If it doesn't work out, the info won't be wasted. */ length = token & ML_MASK; /* match length */ offset = LZ4_readLE16(ip); ip += 2; match = op - offset; assert(match <= op); /* check overflow */ /* Do not deal with overlapping matches. */ if ( (length != ML_MASK) && (offset >= 8) && (dict==withPrefix64k || match >= lowPrefix) ) { /* Copy the match. */ memcpy(op + 0, match + 0, 8); memcpy(op + 8, match + 8, 8); memcpy(op +16, match +16, 2); op += length + MINMATCH; /* Both stages worked, load the next token. */ continue; } /* The second stage didn't work out, but the info is ready. * Propel it right to the point of match copying. */ goto _copy_match; } /* decode literal length */ if (length == RUN_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); if (error == initial_error) { goto _output_error; } if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ } /* copy literals */ cpy = op+length; #if LZ4_FAST_DEC_LOOP safe_literal_copy: #endif LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) { /* We've either hit the input parsing restriction or the output parsing restriction. * If we've hit the input parsing condition then this must be the last sequence. * If we've hit the output parsing condition then we are either using partialDecoding * or we've hit the output parsing condition. */ if (partialDecoding) { /* Since we are partial decoding we may be in this block because of the output parsing * restriction, which is not valid since the output buffer is allowed to be undersized. */ assert(endOnInput); /* If we're in this block because of the input parsing condition, then we must be on the * last sequence (or invalid), so we must check that we exactly consume the input. */ if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; } assert(ip+length <= iend); /* We are finishing in the middle of a literals segment. * Break after the copy. */ if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } assert(ip+length <= iend); } else { /* We must be on the last sequence because of the parsing limitations so check * that we exactly regenerate the original size (must be exact when !endOnInput). */ if ((!endOnInput) && (cpy != oend)) { goto _output_error; } /* We must be on the last sequence (or invalid) because of the parsing limitations * so check that we exactly consume the input and don't overrun the output buffer. */ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } } memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */ ip += length; op += length; /* Necessarily EOF when !partialDecoding. When partialDecoding * it is EOF if we've either filled the output buffer or hit * the input parsing restriction. */ if (!partialDecoding || (cpy == oend) || (ip == iend)) { break; } } else { LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ ip += length; op = cpy; } /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; /* get matchlength */ length = token & ML_MASK; _copy_match: if (length == ML_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); if (error != ok) goto _output_error; if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ } length += MINMATCH; #if LZ4_FAST_DEC_LOOP safe_match_copy: #endif if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { if (partialDecoding) length = MIN(length, (size_t)(oend-op)); else goto _output_error; /* doesn't respect parsing restriction */ } if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ memmove(op, dictEnd - (lowPrefix-match), length); op += length; } else { /* match stretches into both external dictionary and current block */ size_t const copySize = (size_t)(lowPrefix - match); size_t const restSize = length - copySize; memcpy(op, dictEnd - copySize, copySize); op += copySize; if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) *op++ = *copyFrom++; } else { memcpy(op, lowPrefix, restSize); op += restSize; } } continue; } assert(match >= lowPrefix); /* copy match within block */ cpy = op + length; /* partialDecoding : may end anywhere within the block */ assert(op<=oend); if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { size_t const mlen = MIN(length, (size_t)(oend-op)); const BYTE* const matchEnd = match + mlen; BYTE* const copyEnd = op + mlen; if (matchEnd > op) { /* overlap copy */ while (op < copyEnd) { *op++ = *match++; } } else { memcpy(op, match, mlen); } op = copyEnd; if (op == oend) { break; } continue; } if (unlikely(offset<8)) { LZ4_write32(op, 0); /* silence msan warning when offset==0 */ op[0] = match[0]; op[1] = match[1]; op[2] = match[2]; op[3] = match[3]; match += inc32table[offset]; memcpy(op+4, match, 4); match -= dec64table[offset]; } else { memcpy(op, match, 8); match += 8; } op += 8; if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ if (op < oCopyLimit) { LZ4_wildCopy8(op, match, oCopyLimit); match += oCopyLimit - op; op = oCopyLimit; } while (op < cpy) { *op++ = *match++; } } else { memcpy(op, match, 8); if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } } op = cpy; /* wildcopy correction */ } /* end of decoding */ if (endOnInput) { return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ } else { return (int) (((const char*)ip)-src); /* Nb of input bytes read */ } /* Overflow error detected */ _output_error: return (int) (-(((const char*)ip)-src))-1; } } /*===== Instantiate the API decoding functions. =====*/ LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, decode_full_block, noDict, (BYTE*)dest, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { dstCapacity = MIN(targetOutputSize, dstCapacity); return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, endOnInputSize, partial_decode, noDict, (BYTE*)dst, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, decode_full_block, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 0); } /*===== Instantiate a few more decoding cases, used more than once. =====*/ LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, decode_full_block, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 0); } /* Another obsolete API function, paired with the previous one. */ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { /* LZ4_decompress_fast doesn't validate match offsets, * and thus serves well with any prefixed dictionary. */ return LZ4_decompress_fast(source, dest, originalSize); } LZ4_FORCE_O2_GCC_PPC64LE static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, size_t prefixSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, decode_full_block, noDict, (BYTE*)dest-prefixSize, NULL, 0); } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, decode_full_block, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } LZ4_FORCE_O2_GCC_PPC64LE static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, decode_full_block, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } /* The "double dictionary" mode, for use with e.g. ring buffers: the first part * of the dictionary is passed as prefix, and the second via dictStart + dictSize. * These routines are used only once, in LZ4_decompress_*_continue(). */ LZ4_FORCE_INLINE int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, size_t prefixSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, decode_full_block, usingExtDict, (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); } LZ4_FORCE_INLINE int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, size_t prefixSize, const void* dictStart, size_t dictSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, decode_full_block, usingExtDict, (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); } /*===== streaming decompression functions =====*/ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } /*! LZ4_setStreamDecode() : * Use this function to instruct where to find the dictionary. * This function is not necessary if previous data is still available where it was decoded. * Loading a size of 0 is allowed (same effect as no dictionary). * @return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; lz4sd->prefixSize = (size_t) dictSize; lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; lz4sd->externalDict = NULL; lz4sd->extDictSize = 0; return 1; } /*! LZ4_decoderRingBufferSize() : * when setting a ring buffer for streaming decompression (optional scenario), * provides the minimum size of this ring buffer * to be compatible with any source respecting maxBlockSize condition. * Note : in a ring buffer scenario, * blocks are presumed decompressed next to each other. * When not enough space remains for next block (remainingSize < maxBlockSize), * decoding resumes from beginning of ring buffer. * @return : minimum ring buffer size, * or 0 if there is an error (invalid maxBlockSize). */ int LZ4_decoderRingBufferSize(int maxBlockSize) { if (maxBlockSize < 0) return 0; if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; if (maxBlockSize < 16) maxBlockSize = 16; return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); } /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks must still be available at the memory position where they were decoded. If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixSize == 0) { /* The first call, no dictionary yet. */ assert(lz4sd->extDictSize == 0); result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (result <= 0) return result; lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE*)dest + result; } else if (lz4sd->prefixEnd == (BYTE*)dest) { /* They're rolling the current segment. */ if (lz4sd->prefixSize >= 64 KB - 1) result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); else if (lz4sd->extDictSize == 0) result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize); else result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += (size_t)result; lz4sd->prefixEnd += result; } else { /* The buffer wraps around, or they're switching to another buffer. */ lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE*)dest + result; } return result; } LZ4_FORCE_O2_GCC_PPC64LE int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; assert(originalSize >= 0); if (lz4sd->prefixSize == 0) { assert(lz4sd->extDictSize == 0); result = LZ4_decompress_fast(source, dest, originalSize); if (result <= 0) return result; lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } else if (lz4sd->prefixEnd == (BYTE*)dest) { if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) result = LZ4_decompress_fast(source, dest, originalSize); else result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += (size_t)originalSize; lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; result = LZ4_decompress_fast_extDict(source, dest, originalSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } return result; } /* Advanced decoding functions : *_usingDict() : These decoding functions work the same as "_continue" ones, the dictionary must be explicitly provided within parameters */ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { if (dictSize==0) return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { if (dictSize >= 64 KB - 1) { return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); } assert(dictSize >= 0); return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); } assert(dictSize >= 0); return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { if (dictSize==0 || dictStart+dictSize == dest) return LZ4_decompress_fast(source, dest, originalSize); assert(dictSize >= 0); return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } /*=************************************************* * Obsolete Functions ***************************************************/ /* obsolete compression functions */ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } int LZ4_compress(const char* src, char* dest, int srcSize) { return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); } int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); } int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } /* These decompression functions are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } /* Obsolete Streaming functions */ int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } int LZ4_resetStreamState(void* state, char* inputBuffer) { (void)inputBuffer; LZ4_resetStream((LZ4_stream_t*)state); return 0; } void* LZ4_create (char* inputBuffer) { (void)inputBuffer; return LZ4_createStream(); } char* LZ4_slideInputBuffer (void* state) { /* avoid const char * -> char * conversion warning */ return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; } #endif /* LZ4_COMMONDEFS_ONLY */ genomicsdb-0.0~git20231212.9d7ddd0/core/src/codec/external/lz4/lz4.h000066400000000000000000001160071453617025200243630ustar00rootroot00000000000000/* * LZ4 - Fast LZ compression algorithm * Header File * Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - LZ4 homepage : http://www.lz4.org - LZ4 source repository : https://github.com/lz4/lz4 */ #if defined (__cplusplus) extern "C" { #endif #ifndef LZ4_H_2983827168210 #define LZ4_H_2983827168210 /* --- Dependency --- */ #include /* size_t */ /** Introduction LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. The LZ4 compression library provides in-memory compression and decompression functions. It gives full buffer control to user. Compression can be done in: - a single step (described as Simple Functions) - a single step, reusing a context (described in Advanced Functions) - unbounded multiple steps (described as Streaming compression) lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). Decompressing such a compressed block requires additional metadata. Exact metadata depends on exact decompression function. For the typical case of LZ4_decompress_safe(), metadata includes block's compressed size, and maximum bound of decompressed size. Each application is free to encode and pass such metadata in whichever way it wants. lz4.h only handle blocks, it can not generate Frames. Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. The `lz4` CLI can only manage frames. */ /*^*************************************************************** * Export parameters *****************************************************************/ /* * LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL * LZ4LIB_VISIBILITY : * Control library symbols visibility. */ #ifndef LZ4LIB_VISIBILITY # if defined(__GNUC__) && (__GNUC__ >= 4) # define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) # else # define LZ4LIB_VISIBILITY # endif #endif #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) # define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) # define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else # define LZ4LIB_API LZ4LIB_VISIBILITY #endif /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE #define LZ4_QUOTE(str) #str #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ /*-************************************ * Tuning parameter **************************************/ /*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) * Increasing memory usage improves compression ratio. * Reduced memory usage may improve speed, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE # define LZ4_MEMORY_USAGE 14 #endif /*-************************************ * Simple Functions **************************************/ /*! LZ4_compress_default() : * Compresses 'srcSize' bytes from buffer 'src' * into already allocated 'dst' buffer of size 'dstCapacity'. * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). * It also runs faster, so it's a recommended setting. * If the function cannot compress 'src' into a more limited 'dst' budget, * compression stops *immediately*, and the function result is zero. * In which case, 'dst' content is undefined (invalid). * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. * dstCapacity : size of buffer 'dst' (which must be already allocated) * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) * or 0 if compression fails * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). */ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : * compressedSize : is the exact complete size of the compressed block. * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) * If destination buffer is not large enough, decoding will stop and output an error code (negative value). * If the source stream is detected malformed, the function will stop decoding and return a negative result. * Note 1 : This function is protected against malicious data packets : * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, * even if the compressed block is maliciously modified to order the decoder to do these actions. * In such case, the decoder stops immediately, and considers the compressed block malformed. * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. * The implementation is free to send / store / derive this information in whichever way is most beneficial. * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. */ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); /*-************************************ * Advanced Functions **************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) /*! LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) inputSize : max supported value is LZ4_MAX_INPUT_SIZE return : maximum output size in a "worst case" scenario or 0, if input size is incorrect (too large or negative) */ LZ4LIB_API int LZ4_compressBound(int inputSize); /*! LZ4_compress_fast() : Same as LZ4_compress_default(), but allows selection of "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c). */ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_compress_fast_extState() : * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. * Use LZ4_sizeofState() to know how much memory must be allocated, * and allocate it on 8-bytes boundaries (using `malloc()` typically). * Then, provide this buffer as `void* state` to compression function. */ LZ4LIB_API int LZ4_sizeofState(void); LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_compress_destSize() : * Reverse the logic : compresses as much data as possible from 'src' buffer * into already allocated buffer 'dst', of size >= 'targetDestSize'. * This function either compresses the entire 'src' content into 'dst' if it's large enough, * or fill 'dst' buffer completely with as much data as possible from 'src'. * note: acceleration parameter is fixed to "default". * * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. * New value is necessarily <= input value. * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) * or 0 if compression fails. */ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); /*! LZ4_decompress_safe_partial() : * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', * into destination buffer 'dst' of size 'dstCapacity'. * Up to 'targetOutputSize' bytes will be decoded. * The function stops decoding on reaching this objective, * which can boost performance when only the beginning of a block is required. * * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity) * If source stream is detected malformed, function returns a negative result. * * Note : @return can be < targetOutputSize, if compressed block contains less data. * * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity, * and expects targetOutputSize <= dstCapacity. * It effectively stops decoding on reaching targetOutputSize, * so dstCapacity is kind of redundant. * This is because in a previous version of this function, * decoding operation would not "break" a sequence in the middle. * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize, * it could write more bytes, though only up to dstCapacity. * Some "margin" used to be required for this operation to work properly. * This is no longer necessary. * The function nonetheless keeps its signature, in an effort to not break API. */ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); /*-********************************************* * Streaming Compression Functions ***********************************************/ typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); /*! LZ4_resetStream_fast() : v1.9.0+ * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks * (e.g., LZ4_compress_fast_continue()). * * An LZ4_stream_t must be initialized once before usage. * This is automatically done when created by LZ4_createStream(). * However, should the LZ4_stream_t be simply declared on stack (for example), * it's necessary to initialize it first, using LZ4_initStream(). * * After init, start any new stream with LZ4_resetStream_fast(). * A same LZ4_stream_t can be re-used multiple times consecutively * and compress multiple streams, * provided that it starts each new stream with LZ4_resetStream_fast(). * * LZ4_resetStream_fast() is much faster than LZ4_initStream(), * but is not compatible with memory regions containing garbage data. * * Note: it's only useful to call LZ4_resetStream_fast() * in the context of streaming compression. * The *extState* functions perform their own resets. * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. */ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); /*! LZ4_loadDict() : * Use this function to reference a static dictionary into LZ4_stream_t. * The dictionary must remain available during compression. * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. * The same dictionary will have to be loaded on decompression side for successful decoding. * Dictionary are useful for better compression of small data (KB range). * While LZ4 accept any input as dictionary, * results are generally better when using Zstandard's Dictionary Builder. * Loading a size of 0 is allowed, and is the same as reset. * @return : loaded dictionary size, in bytes (necessarily <= 64 KB) */ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : * Compress 'src' content using data from previously compressed blocks, for better compression ratio. * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * * @return : size of compressed block * or 0 if there is an error (typically, cannot fit into 'dst'). * * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. * Each block has precise boundaries. * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. * * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! * * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. * Make sure that buffers are separated, by at least one byte. * This construction ensures that each block only depends on previous block. * * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. * * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_saveDict() : * If last 64KB data cannot be guaranteed to remain available at its current memory location, * save it into a safer place (char* safeBuffer). * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. */ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); /*-********************************************** * Streaming Decompression Functions * Bufferless synchronous API ************************************************/ typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : * creation / destruction of streaming decompression tracking context. * A tracking context can be re-used multiple times. */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /*! LZ4_setStreamDecode() : * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. * Use this function to start decompression of a new stream of blocks. * A dictionary can optionally be set. Use NULL or size 0 for a reset order. * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. * @return : 1 if OK, 0 if error */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); /*! LZ4_decoderRingBufferSize() : v1.8.2+ * Note : in a ring buffer scenario (optional), * blocks are presumed decompressed next to each other * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), * at which stage it resumes from beginning of ring buffer. * When setting such a ring buffer for streaming decompression, * provides the minimum size of this ring buffer * to be compatible with any source respecting maxBlockSize condition. * @return : minimum ring buffer size, * or 0 if there is an error (invalid maxBlockSize). */ LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); #define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ /*! LZ4_decompress_*_continue() : * These decoding functions allow decompression of consecutive blocks in "streaming" mode. * A block is an unsplittable entity, it must be presented entirely to a decompression function. * Decompression functions only accepts one block at a time. * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. * If less than 64KB of data has been decoded, all the data must be present. * * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. * In which case, encoding and decoding buffers do not need to be synchronized. * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. * - Synchronized mode : * Decompression buffer size is _exactly_ the same as compression buffer size, * and follows exactly same update rule (block boundaries at same positions), * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including small ones ( < 64 KB). * * Whenever these conditions are not possible, * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_*_usingDict() : * These decoding functions work the same as * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. * Performance tip : Decompression speed can be substantially increased * when dst == dictStart + dictSize. */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); #endif /* LZ4_H_2983827168210 */ /*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! ***************************************/ /*-**************************************************************************** * Experimental section * * Symbols declared in this section must be considered unstable. Their * signatures or semantics may change, or they may be removed altogether in the * future. They are therefore only safe to depend on when the caller is * statically linked against the library. * * To protect against unsafe usage, not only are the declarations guarded, * the definitions are hidden by default * when building LZ4 as a shared/dynamic library. * * In order to access these declarations, * define LZ4_STATIC_LINKING_ONLY in your application * before including LZ4's headers. * * In order to make their implementations accessible dynamically, you must * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. ******************************************************************************/ #ifdef LZ4_STATIC_LINKING_ONLY #ifndef LZ4_STATIC_3504398509 #define LZ4_STATIC_3504398509 #ifdef LZ4_PUBLISH_STATIC_FUNCTIONS #define LZ4LIB_STATIC_API LZ4LIB_API #else #define LZ4LIB_STATIC_API #endif /*! LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). * * Using this variant avoids an expensive initialization step. * It is only safe to call if the state buffer is known to be correctly initialized already * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). * From a high level, the difference is that * this function initializes the provided state with a call to something like LZ4_resetStream_fast() * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). */ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_attach_dictionary() : * This is an experimental API that allows * efficient use of a static dictionary many times. * * Rather than re-loading the dictionary buffer into a working context before * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a * working LZ4_stream_t, this function introduces a no-copy setup mechanism, * in which the working stream references the dictionary stream in-place. * * Several assumptions are made about the state of the dictionary stream. * Currently, only streams which have been prepared by LZ4_loadDict() should * be expected to work. * * Alternatively, the provided dictionaryStream may be NULL, * in which case any existing dictionary stream is unset. * * If a dictionary is provided, it replaces any pre-existing stream history. * The dictionary contents are the only history that can be referenced and * logically immediately precede the data compressed in the first subsequent * compression call. * * The dictionary will only remain attached to the working stream through the * first compression call, at the end of which it is cleared. The dictionary * stream (and source buffer) must remain in-place / accessible / unchanged * through the completion of the first compression call on the stream. */ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); /*! In-place compression and decompression * * It's possible to have input and output sharing the same buffer, * for highly contrained memory environments. * In both cases, it requires input to lay at the end of the buffer, * and decompression to start at beginning of the buffer. * Buffer size must feature some margin, hence be larger than final size. * * |<------------------------buffer--------------------------------->| * |<-----------compressed data--------->| * |<-----------decompressed size------------------>| * |<----margin---->| * * This technique is more useful for decompression, * since decompressed size is typically larger, * and margin is short. * * In-place decompression will work inside any buffer * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). * This presumes that decompressedSize > compressedSize. * Otherwise, it means compression actually expanded data, * and it would be more efficient to store such data with a flag indicating it's not compressed. * This can happen when data is not compressible (already compressed, or encrypted). * * For in-place compression, margin is larger, as it must be able to cope with both * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, * and data expansion, which can happen when input is not compressible. * As a consequence, buffer size requirements are much higher, * and memory savings offered by in-place compression are more limited. * * There are ways to limit this cost for compression : * - Reduce history size, by modifying LZ4_DISTANCE_MAX. * Note that it is a compile-time constant, so all compressions will apply this limit. * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, * so it's a reasonable trick when inputs are known to be small. * - Require the compressor to deliver a "maximum compressed size". * This is the `dstCapacity` parameter in `LZ4_compress*()`. * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, * in which case, the return code will be 0 (zero). * The caller must be ready for these cases to happen, * and typically design a backup scheme to send data uncompressed. * The combination of both techniques can significantly reduce * the amount of margin required for in-place compression. * * In-place compression can work in any buffer * which size is >= (maxCompressedSize) * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, * so it's possible to reduce memory requirements by playing with them. */ #define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) #define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ #ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ # define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif #define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ #define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ #endif /* LZ4_STATIC_3504398509 */ #endif /* LZ4_STATIC_LINKING_ONLY */ #ifndef LZ4_H_98237428734687 #define LZ4_H_98237428734687 /*-************************************************************ * PRIVATE DEFINITIONS ************************************************************** * Do not use these definitions directly. * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. * Accessing members will expose code to API and/or ABI break in future versions of the library. **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #include typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { uint32_t hashTable[LZ4_HASH_SIZE_U32]; uint32_t currentOffset; uint16_t dirty; uint16_t tableType; const uint8_t* dictionary; const LZ4_stream_t_internal* dictCtx; uint32_t dictSize; }; typedef struct { const uint8_t* externalDict; size_t extDictSize; const uint8_t* prefixEnd; size_t prefixSize; } LZ4_streamDecode_t_internal; #else typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { unsigned int hashTable[LZ4_HASH_SIZE_U32]; unsigned int currentOffset; unsigned short dirty; unsigned short tableType; const unsigned char* dictionary; const LZ4_stream_t_internal* dictCtx; unsigned int dictSize; }; typedef struct { const unsigned char* externalDict; const unsigned char* prefixEnd; size_t extDictSize; size_t prefixSize; } LZ4_streamDecode_t_internal; #endif /*! LZ4_stream_t : * information structure to track an LZ4 stream. * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. * The structure definition can be convenient for static allocation * (on stack, or as part of larger structure). * Init this structure with LZ4_initStream() before first use. * note : only use this definition in association with static linking ! * this definition is not API/ABI safe, and may change in a future version. */ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ ) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { unsigned long long table[LZ4_STREAMSIZE_U64]; LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ /*! LZ4_initStream() : v1.9.0+ * An LZ4_stream_t structure must be initialized at least once. * This is automatically done when invoking LZ4_createStream(), * but it's not when the structure is simply declared on stack (for example). * * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. * It can also initialize any arbitrary buffer of sufficient size, * and will @return a pointer of proper type upon initialization. * * Note : initialization fails if size and alignment conditions are not respected. * In which case, the function will @return NULL. * Note2: An LZ4_stream_t structure guarantees correct alignment and size. * Note3: Before v1.9.0, use LZ4_resetStream() instead */ LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); /*! LZ4_streamDecode_t : * information structure to track an LZ4 stream during decompression. * init this structure using LZ4_setStreamDecode() before first use. * note : only use in association with static linking ! * this definition is not API/ABI safe, * and may change in a future version ! */ #define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ /*-************************************ * Obsolete Functions **************************************/ /*! Deprecation warnings * * Deprecated functions make the compiler generate a warning when invoked. * This is meant to invite users to update their source code. * Should deprecation warnings be a problem, it is generally possible to disable them, * typically with -Wno-deprecated-declarations for gcc * or _CRT_SECURE_NO_WARNINGS in Visual. * * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS * before including the header file. */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] # elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) # else # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") # define LZ4_DEPRECATED(message) # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Obsolete decompression functions */ LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); /* Obsolete streaming functions; degraded functionality; do not use! * * In order to perform streaming compression, these functions depended on data * that is no longer tracked in the state. They have been preserved as well as * possible: using them will still produce a correct output. However, they don't * actually retain any history between compression calls. The compression ratio * achieved will therefore be no better than compressing each chunk * independently. */ LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); /*! LZ4_decompress_fast() : **unsafe!** * These functions used to be faster than LZ4_decompress_safe(), * but it has changed, and they are now slower than LZ4_decompress_safe(). * This is because LZ4_decompress_fast() doesn't know the input size, * and therefore must progress more cautiously in the input buffer to not read beyond the end of block. * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. * * The last remaining LZ4_decompress_fast() specificity is that * it can decompress a block without knowing its compressed size. * Such functionality could be achieved in a more secure manner, * by also providing the maximum size of input buffer, * but it would require new prototypes, and adaptation of the implementation to this new use case. * * Parameters: * originalSize : is the uncompressed size to regenerate. * `dst` must be already allocated, its size must be >= 'originalSize' bytes. * @return : number of bytes read from source buffer (== compressed size). * The function expects to finish at block's end exactly. * If the source stream is detected malformed, the function stops decoding and returns a negative result. * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. * Also, since match offsets are not validated, match reads from 'src' may underflow too. * These issues never happen if input (compressed) data is correct. * But they may happen if input data is invalid (error or intentional tampering). * As a consequence, use these functions in trusted environments with trusted data **only**. */ LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); /*! LZ4_resetStream() : * An LZ4_stream_t structure must be initialized at least once. * This is done with LZ4_initStream(), or LZ4_resetStream(). * Consider switching to LZ4_initStream(), * invoking LZ4_resetStream() will trigger deprecation warnings in the future. */ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); #endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) } #endif genomicsdb-0.0~git20231212.9d7ddd0/core/src/expressions/000077500000000000000000000000001453617025200224465ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/expressions/expression.cc000066400000000000000000000525411453617025200251630ustar00rootroot00000000000000/** * @file expression.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2019, 2022-2023 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the Expression class. */ #include "array_read_state.h" #include "error.h" #include "expression.h" #include "tiledb.h" #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #define EXPRESSION_ERROR(MSG) TILEDB_ERROR(TILEDB_EXPR_ERRMSG, MSG, tiledb_expr_errmsg) /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_expr_errmsg = ""; int Expression::init(const std::vector& attribute_ids, const ArraySchema* array_schema) { array_schema_ = array_schema; for (std::vector::const_iterator it = attribute_ids.begin(); it != attribute_ids.end(); it++) { attributes_.push_back(array_schema_->attribute(*it)); } if (array_schema_->dense()) { EXPRESSION_ERROR("Expression parsing for dense arrays not yet implemented"); return TILEDB_EXPR_ERR; } if (expression_.size() != 0) { parser_->EnableOptimizer(true); // muparser takes full control of DefineFun/Oprt pointers // Do not release them. parser_->DefineFun(new SplitCompare); parser_->DefineOprt(new OprtSplitCompare); parser_->DefineFun(new Resolve); parser_->DefineOprt(new OprtCompareAll); parser_->DefineFun(new IsHomRef); parser_->DefineFun(new IsHomAlt); parser_->DefineFun(new IsHet); // Setup muparserx variables for the attributes try { // TODO: Move this block to GenomicsDB as they are GenomicsDB aliases! if (array_schema_->dim_num() == 2 && array_schema_->cell_order() == TILEDB_COL_MAJOR) { //muparserx does not have a unary negation operator, so use regex to replace !IS* aliases std::regex alias("(ROW)|(POS)|(!ISHOMREF)|(!ISHOMALT)|(!ISHET)|(ISHOMREF)|(ISHOMALT)|(ISHET)"); for (std::smatch match; std::regex_search(expression_, match, alias); ) { if (match[0] == "ROW") { expression_ = match.prefix().str() + "__coords[0]" + match.suffix().str(); } else if (match[0] == "POS") { expression_ = match.prefix().str() + "__coords[1]" + match.suffix().str(); } else if (match[0] == "ISHOMREF") { expression_ = match.prefix().str() + "ishomref(GT)" + match.suffix().str(); } else if (match[0] == "ISHOMALT") { expression_ = match.prefix().str() + "ishomalt(GT)" + match.suffix().str(); } else if (match[0] == "ISHET") { expression_ = match.prefix().str() + "ishet(GT)" + match.suffix().str(); } else if (match[0] == "!ISHOMREF") { expression_ = match.prefix().str() + "(ishomref(GT) == false)" + match.suffix().str(); } else if (match[0] == "!ISHOMALT") { expression_ = match.prefix().str() + "(ishomalt(GT) == false)" + match.suffix().str(); } else if (match[0] == "!ISHET") { expression_ = match.prefix().str() + "(ishet(GT) == false)" + match.suffix().str(); } } } parser_->SetExpr(expression_); // Get map of all variables used in the expression mup::var_maptype vmap = parser_->GetExprVar(); for (mup::var_maptype::iterator item = vmap.begin(); item!=vmap.end(); ++item) { if (std::find(attributes_.begin(), attributes_.end(), item->first) != attributes_.end()) { add_attribute(item->first); } else { EXPRESSION_ERROR("Attribute " + item->first + " in expression filter not present in the array schema"); return TILEDB_EXPR_ERR; } } } catch (mup::ParserError const &e) { EXPRESSION_ERROR("Parser SetExpr error for expression '" + expression_ + "': " + e.GetMsg()); return TILEDB_EXPR_ERR; } last_processed_buffer_index_.resize(attributes_.size()); } is_initialized = true; return TILEDB_EXPR_OK; } void Expression::add_attribute(std::string name) { int attribute_id = array_schema_->attribute_id(name); int attribute_type = array_schema_->type(attribute_id); int attribute_cell_val_num = get_cell_val_num(name); if (attribute_cell_val_num == TILEDB_VAR_NUM) { attribute_cell_val_num = 0; } switch (attribute_type) { case TILEDB_CHAR: { if (attribute_cell_val_num == 1) { mup::Value x_int(mup::int_type(0)); attribute_map_.insert(std::make_pair(name, x_int)); } else { mup::Value x_str(mup::string_type("")); attribute_map_.insert(std::make_pair(name, x_str)); } break; } case TILEDB_INT8: case TILEDB_INT16: case TILEDB_INT32: case TILEDB_INT64: case TILEDB_UINT8: case TILEDB_UINT16: case TILEDB_UINT32: case TILEDB_UINT64: { if (attribute_cell_val_num == 1) { mup::Value x_int(mup::int_type(0)); attribute_map_.insert(std::make_pair(name, x_int)); } else { mup::Value x_int_array(mup::int_type(attribute_cell_val_num), mup::int_type(0)); attribute_map_.insert(std::make_pair(name, x_int_array)); } break; } case TILEDB_FLOAT32: case TILEDB_FLOAT64: { if (attribute_cell_val_num == 1) { mup::Value x_float(mup::float_type(0.0)); attribute_map_.insert(std::make_pair(name, x_float)); } else { mup::Value x_float_array(mup::int_type(attribute_cell_val_num), mup::float_type(0.0)); attribute_map_.insert(std::make_pair(name, x_float_array)); } break; } } parser_->DefineVar(name, (mup::Variable)&(attribute_map_[name])); } // Get a map of all variables used by muParserX void print_parser_varmap(mup::ParserX *parser) { #if 0 mup::var_maptype vmap = parser->GetVar(); std::cerr << "Map of all variables used by parser" << std::endl; for (mup::var_maptype::iterator item = vmap.begin(); item!=vmap.end(); ++item) std::cerr << item->first << "=" << (mup::Variable&)(*(item->second)) << std::endl; std::cerr << "Map of all variables used by parser - END" << std::endl; #endif } // Get a map of all the variables used in the expression void print_parser_expr_varmap(mup::ParserX *parser) { #if 0 mup::var_maptype vmap = parser->GetExprVar(); std::cerr << "Map of all variables used in the expression to the parser" << std::endl; for (mup::var_maptype::iterator item = vmap.begin(); item!=vmap.end(); ++item) std::cerr << item->first << "=" << (mup::Variable&)(*(item->second)) << std::endl; std::cerr << "Map of all variables used in the expression to the parser - END" << std::endl; #endif } struct EmptyValueException : public std::exception { const char* what () const throw () { return "NYI: Filter expressions do not handle empty values yet"; } }; template T get_value(const void *buffer, const uint64_t offset) { T val = *(reinterpret_cast(buffer) + offset); if ((typeid(T) == typeid(char) && val == TILEDB_EMPTY_CHAR) || (typeid(T) == typeid(int) && val == TILEDB_EMPTY_INT32) || (typeid(T) == typeid(float) && val == TILEDB_EMPTY_FLOAT32)) { throw EmptyValueException(); } return val; } inline mup::Value get_single_cell_value(const int attribute_type, void** buffers, const uint64_t buffer_index, const uint64_t position) { switch (attribute_type) { case TILEDB_CHAR: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_UINT8: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_UINT16: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_UINT32: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_UINT64: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_INT8: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_INT16: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_INT32: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_INT64: return mup::int_type(get_value(buffers[buffer_index], position)); case TILEDB_FLOAT32: return mup::float_type(get_value(buffers[buffer_index], position)); case TILEDB_FLOAT64: return mup::float_type(get_value(buffers[buffer_index], position)); default: throw std::range_error("Attribute Type " + std::to_string(attribute_type) + " not supported in expressions"); } } void Expression::assign_single_cell_value(const int attribute_id, void** buffers, const uint64_t buffer_index, const uint64_t position) { auto& attribute_name = array_schema_->attribute(attribute_id); attribute_map_[attribute_name] = get_single_cell_value(array_schema_->type(attribute_id), buffers, buffer_index, position); } void Expression::assign_fixed_cell_values(const int attribute_id, void** buffers, const uint64_t buffer_index, const uint64_t position) { auto& attribute_name = array_schema_->attribute(attribute_id); auto attribute_type = array_schema_->type(attribute_id); long num_cells = get_cell_val_num(attribute_name); switch (attribute_type) { case TILEDB_CHAR: { attribute_map_[attribute_name] = mup::string_type(get_value(buffers[buffer_index], position*num_cells), num_cells); break; } default: { auto& x_array = attribute_map_[attribute_name]; for (auto i=0u; itype(attribute_id), buffers, buffer_index, position*num_cells+i); } } } } // returns offset and length for the cell under consideration template std::pair get_var_cell_info(void** buffers, size_t* buffer_sizes, const uint64_t buffer_index, const uint64_t buffer_position) { size_t offset = get_value(buffers[buffer_index], buffer_position); size_t length; if ((buffer_position+1) < (buffer_sizes[buffer_index]/sizeof(size_t))) { length = get_value(buffers[buffer_index], buffer_position+1)-offset; } else { length=buffer_sizes[buffer_index+1]-offset; } return std::make_pair(offset/sizeof(T), length/sizeof(T)); } void Expression::assign_var_cell_values(const int attribute_id, void** buffers, size_t *buffer_sizes, const uint64_t buffer_index, const uint64_t position) { auto& attribute_name = array_schema_->attribute(attribute_id); auto attribute_type = array_schema_->type(attribute_id); switch (attribute_type) { case TILEDB_CHAR: { std::pair info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); attribute_map_[attribute_name] = mup::string_type(std::string((char *)buffers[buffer_index+1]+info.first, info.second)); break; } default: { std::pair info; switch (attribute_type) { case TILEDB_UINT8: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_UINT16: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_UINT32: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_UINT64: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_INT8: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_INT16: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_INT32: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_INT64: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_FLOAT32: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; case TILEDB_FLOAT64: info = get_var_cell_info(buffers, buffer_sizes, buffer_index, position); break; default: throw std::range_error("Attribute Type " + std::to_string(attribute_type) + " not supported in expressions"); } if (attribute_map_[attribute_name].GetRows() != (int)info.second) { parser_->RemoveVar(attribute_name); assert(!parser_->IsVarDefined(attribute_name)); mup::Value x_array(mup::int_type(info.second), mup::int_type(0)); attribute_map_[attribute_name] = std::move(x_array); parser_->DefineVar(attribute_name, (mup::Variable)&(attribute_map_[attribute_name])); } for (auto i=0u; i& positions) { return evaluate_cell(buffers, buffer_sizes, positions.data()); } int Expression::evaluate_cell(void** buffers, size_t* buffer_sizes, int64_t* positions) { if (expression_.empty()) { return true; } if (!is_initialized) { EXPRESSION_ERROR("Initialization not completed"); return TILEDB_EXPR_ERR; } for (auto i = 0u, j = 0u; i < attributes_.size(); i++, j++) { assert(positions[i] >= 0); uint64_t position = (uint64_t)positions[i]; int attribute_id = array_schema_->attribute_id(attributes_[i]); if (attribute_map_.find(attributes_[i]) != attribute_map_.end()) { try { switch (get_cell_val_num(attributes_[i])) { case 1: assign_single_cell_value(attribute_id, buffers, j, position); break; case TILEDB_VAR_NUM : assign_var_cell_values(attribute_id, buffers, buffer_sizes, j, position); break; default: assign_fixed_cell_values(attribute_id, buffers, j, position); } } catch (EmptyValueException& e) { EXPRESSION_ERROR(e.what()); return true; } } // Increment buffer index for variable types if (array_schema_->cell_size(attribute_id) == TILEDB_VAR_SIZE) { j++; } } bool keep_cell = true; try { mup::Value value = parser_->Eval(); #if 0 // print the result mup::console() << "Ret=" << value << std::endl; #endif // TODO: Only supports expressions evaluating to booleans for now. if (value.GetType() == 'b') { keep_cell = value.GetBool(); } else { EXPRESSION_ERROR("Only expressions evaluating to booleans is supported"); return TILEDB_EXPR_ERR; } } catch (mup::ParserError const &e) { EXPRESSION_ERROR("Parser evaluate error, possibly due to bad filter expression: " + "\n\t" + e.GetMsg()); return TILEDB_EXPR_ERR; } return keep_cell; } int get_num_cells(const ArraySchema *array_schema, int attribute_id, size_t* buffer_sizes, int buffer_index) { if (array_schema->cell_size(attribute_id) == TILEDB_VAR_SIZE) { return buffer_sizes[buffer_index]/TILEDB_CELL_VAR_OFFSET_SIZE; } else { return buffer_sizes[buffer_index]/array_schema->cell_size(attribute_id); } } /** * Only used by the unit tests now */ int Expression::evaluate(void** buffers, size_t* buffer_sizes) { if (expression_.empty()) { return TILEDB_EXPR_OK; } if (!is_initialized) { EXPRESSION_ERROR("Initialization not completed"); return TILEDB_EXPR_ERR; } // Get minimum number of cells in buffers for evaluation to account for overflow. size_t number_of_cells = 0; for (auto i = 0u, j = 0u; i < attributes_.size(); i++, j++) { int attribute_id = array_schema_->attribute_id(attributes_[i]); size_t ncells = 0; if (buffer_sizes[j] != 0) { ncells = get_num_cells(array_schema_, attribute_id, buffer_sizes, j); last_processed_buffer_index_[i] = 0; } number_of_cells = (!number_of_cells || ncells < number_of_cells)?ncells:number_of_cells; // Increment buffer index for variable types if (array_schema_->cell_size(attribute_id) == TILEDB_VAR_SIZE) j++; } if (number_of_cells == 0) { return TILEDB_EXPR_OK; } std::vector cells_to_be_dropped; print_parser_varmap(parser_); print_parser_expr_varmap(parser_); for (auto i_cell = 0u; i_cell < number_of_cells; i_cell++) { int rc = evaluate_cell(buffers, buffer_sizes, last_processed_buffer_index_); if (rc == TILEDB_EXPR_ERR) { return TILEDB_EXPR_ERR; } else if (!rc) { cells_to_be_dropped.push_back(i_cell); } for (auto i = 0u; i cells_to_be_dropped) { std::map adjust_offsets; std::vector num_cells(attributes_.size()); // Initialize num_cells for all attributes as the buffer_sizes are being updated in place for (auto i=0u, j=0u; i < attributes_.size(); i++, j++) { num_cells[i] = buffer_sizes[j]/get_cell_size(attributes_[i]); if (get_cell_val_num(attributes_[i]) == TILEDB_VAR_NUM) j++; } auto max_num_cells = std::max_element(num_cells.begin(), num_cells.end()); for (auto current_cell=0ul, next_cell=0ul; next_cell < *max_num_cells; current_cell++, next_cell++) { size_t reduce_by = 0; bool next_cell_dropped = false; do { next_cell_dropped = std::find(cells_to_be_dropped.begin(), cells_to_be_dropped.end(), next_cell) != cells_to_be_dropped.end(); if (next_cell_dropped && next_cell < number_of_cells) { reduce_by++; next_cell++; } } while (next_cell_dropped); for (auto i=0u, j=0u; i < attributes_.size(); i++, j++) { int cell_val_num = get_cell_val_num(attributes_[i]); size_t cell_size = get_cell_size(attributes_[i]); if (current_cell != next_cell && next_cell < num_cells[i]) { void *next = static_cast(buffers[j])+cell_size*next_cell; void *current = static_cast(buffers[j])+cell_size*current_cell; if (cell_val_num == TILEDB_VAR_NUM) { assert(cell_size == sizeof(size_t)); // Initialization if (adjust_offsets.find(j) == adjust_offsets.end()) adjust_offsets[j] = 0; auto var_cell_type_size = get_var_cell_type_size(attributes_[i]); size_t next_length = 0; // Length of next cell in cells if ((next_cell+1) < num_cells[i]) { next_length = *(reinterpret_cast(next)+1) - *(reinterpret_cast(next)); } else { next_length = buffer_sizes[j+1] - *(reinterpret_cast(next)); } void *var_next = offset_pointer(attributes_[i], buffers[j+1], (*(reinterpret_cast(next)))/var_cell_type_size); void *var_current = offset_pointer(attributes_[i], buffers[j+1], adjust_offsets[j]); // Shift cell contents into (dropped)current contents memmove(var_current, var_next, next_length); // Adjust the current cell's offsets size_t adjust_offset = adjust_offsets[j]*var_cell_type_size; memmove(current, &adjust_offset, cell_size); adjust_offsets[j] += next_length/var_cell_type_size; } else { // Move next cell contents into current contents memmove(current, next, cell_size); } } if (reduce_by) buffer_sizes[j] -= reduce_by*cell_size; if (cell_val_num == TILEDB_VAR_NUM) j++; } } // Fixup buffer_sizes for attributes with variable cell size for (auto i=0u, j=0u; i < attributes_.size(); i++, j++) { if (get_cell_val_num(attributes_[i]) == TILEDB_VAR_NUM) { buffer_sizes[j+1] = adjust_offsets[j]*get_var_cell_type_size(attributes_[i]); j++; } } } genomicsdb-0.0~git20231212.9d7ddd0/core/src/fragment/000077500000000000000000000000001453617025200216675ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/fragment/book_keeping.cc000066400000000000000000000664601453617025200246460ustar00rootroot00000000000000/** * @file book_keeping.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2021-2022 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the BookKeeping class. */ #include "book_keeping.h" #include "storage_fs.h" #include "utils.h" #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_BK_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_bk_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ BookKeeping::BookKeeping( const ArraySchema* array_schema, bool dense, const std::string& fragment_name, int mode) : array_schema_(array_schema), dense_(dense), fragment_name_(fragment_name), mode_(mode) { domain_ = NULL; non_empty_domain_ = NULL; // Prepare file name filename_ = StorageFS::append_paths(fragment_name_, std::string(TILEDB_BOOK_KEEPING_FILENAME) + TILEDB_FILE_SUFFIX + TILEDB_GZIP_SUFFIX); } BookKeeping::~BookKeeping() { if(domain_ != NULL) free(domain_); if(non_empty_domain_ != NULL) free(non_empty_domain_); int64_t mbr_num = mbrs_.size(); for(int64_t i=0; i& BookKeeping::bounding_coords() const { return bounding_coords_; } int64_t BookKeeping::cell_num(int64_t tile_pos) const { if(dense_) { return array_schema_->cell_num_per_tile(); } else { int64_t tile_num = this->tile_num(); if(tile_pos != tile_num-1) return array_schema_->capacity(); else return last_tile_cell_num(); } } bool BookKeeping::dense() const { return dense_; } const void* BookKeeping::domain() const { return domain_; } int64_t BookKeeping::last_tile_cell_num() const { return last_tile_cell_num_; } const std::vector& BookKeeping::mbrs() const { return mbrs_; } const void* BookKeeping::non_empty_domain() const { return non_empty_domain_; } inline bool BookKeeping::read_mode() const { return array_read_mode(mode_); } int64_t BookKeeping::tile_num() const { if(dense_) { return array_schema_->tile_num(domain_); } else { return mbrs_.size(); } } const std::vector >& BookKeeping::tile_offsets() const { return tile_offsets_; } const std::vector >& BookKeeping::tile_var_offsets() const { return tile_var_offsets_; } const std::vector >& BookKeeping::tile_var_sizes() const { return tile_var_sizes_; } inline bool BookKeeping::write_mode() const { return array_write_mode(mode_); } /* ****************************** */ /* MUTATORS */ /* ****************************** */ void BookKeeping::append_bounding_coords(const void* bounding_coords) { // For easy reference size_t bounding_coords_size = 2*array_schema_->coords_size(); // Copy and append MBR void* new_bounding_coords = malloc(bounding_coords_size); memcpy(new_bounding_coords, bounding_coords, bounding_coords_size); bounding_coords_.push_back(new_bounding_coords); } void BookKeeping::append_mbr(const void* mbr) { // For easy reference size_t mbr_size = 2*array_schema_->coords_size(); // Copy and append MBR void* new_mbr = malloc(mbr_size); memcpy(new_mbr, mbr, mbr_size); mbrs_.push_back(new_mbr); } void BookKeeping::append_tile_offset( int attribute_id, size_t step) { tile_offsets_[attribute_id].push_back(next_tile_offsets_[attribute_id]); size_t new_offset = tile_offsets_[attribute_id].back() + step; next_tile_offsets_[attribute_id] = new_offset; } void BookKeeping::append_tile_var_offset( int attribute_id, size_t step) { tile_var_offsets_[attribute_id].push_back( next_tile_var_offsets_[attribute_id]); size_t new_offset = tile_var_offsets_[attribute_id].back() + step; next_tile_var_offsets_[attribute_id] = new_offset; } void BookKeeping::append_tile_var_size( int attribute_id, size_t size) { tile_var_sizes_[attribute_id].push_back(size); } /* FORMAT: * non_empty_domain_size(size_t) non_empty_domain(void*) * mbr_num(int64_t) * mbr_#1(void*) mbr_#2(void*) ... * bounding_coords_num(int64_t) * bounding_coords_#1(void*) bounding_coords_#2(void*) ... * tile_offsets_attr#0_num(int64_t) * tile_offsets_attr#0_#1 (off_t) tile_offsets_attr#0_#2 (off_t) ... * ... * tile_offsets_attr#_num(int64_t) * tile_offsets_attr#_#1(off_t) * tile_offsets_attr#_#2 (off_t) ... * tile_var_offsets_attr#0_num(int64_t) * tile_var_offsets_attr#0_#1 (off_t) tile_var_offsets_attr#0_#2 (off_t) ... * ... * tile_var_offsets_attr#_num(int64_t) * tile_var_offsets_attr#_#1 (off_t) * tile_var_offsets_attr#_#2 (off_t) ... * tile_var_sizes_attr#0_num(int64_t) * tile_var_sizes_attr#0_#1(size_t) tile_sizes_attr#0_#2 (size_t) ... * ... * tile_var_sizes_attr#_num(int64_t) * tile_var_sizes__attr#_#1(size_t) * tile_var_sizes_attr#_#2 (size_t) ... * last_tile_cell_num(int64_t) */ int BookKeeping::finalize(StorageFS *fs) { // Nothing to do in READ mode if(read_mode()) return TILEDB_BK_OK; // Do nothing if the fragment directory does not exist (fragment empty) if(!is_dir(fs, fragment_name_)) return TILEDB_BK_OK; // Create StorageBuffer to serialize book_keeping content buffer_ = new CompressedStorageBuffer(fs, filename_, upload_uncompressed_size_, /*is_read*/false, TILEDB_GZIP, TILEDB_COMPRESSION_LEVEL_GZIP); // Write non-empty domain if(flush_non_empty_domain() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Write MBRs if(flush_mbrs() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Write bounding coordinates if(flush_bounding_coords() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Write tile offsets if(flush_tile_offsets() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Write variable tile offsets if(flush_tile_var_offsets() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Write variable tile sizes if(flush_tile_var_sizes() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Write cell number of the last tile if(flush_last_tile_cell_num() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Free up StorageBuffer buffer_->finalize(); delete buffer_; buffer_ = 0; // Success return TILEDB_BK_OK; } int BookKeeping::init(const void* non_empty_domain) { // For easy reference int attribute_num = array_schema_->attribute_num(); // Sanity check assert(non_empty_domain_ == NULL); assert(domain_ == NULL); // Set non-empty domain size_t domain_size = 2*array_schema_->coords_size(); non_empty_domain_ = malloc(domain_size); if(non_empty_domain == NULL) memcpy(non_empty_domain_, array_schema_->domain(), domain_size); else memcpy(non_empty_domain_, non_empty_domain, domain_size); // Set expanded domain domain_ = malloc(domain_size); memcpy(domain_, non_empty_domain_, domain_size); array_schema_->expand_domain(domain_); // Set last tile cell number last_tile_cell_num_ = 0; // Initialize tile offsets tile_offsets_.resize(attribute_num+1); next_tile_offsets_.resize(attribute_num+1); for(int i=0; i_num(int64_t) * tile_offsets_attr#_#1(off_t) * tile_offsets_attr#_#2 (off_t) ... * tile_var_offsets_attr#0_num(int64_t) * tile_var_offsets_attr#0_#1 (off_t) tile_var_offsets_attr#0_#2 (off_t) ... * ... * tile_var_offsets_attr#_num(int64_t) * tile_var_offsets_attr#_#1 (off_t) * tile_var_offsets_attr#_#2 (off_t) ... * tile_var_sizes_attr#0_num(int64_t) * tile_var_sizes_attr#0_#1(size_t) tile_sizes_attr#0_#2 (size_t) ... * ... * tile_var_sizes_attr#_num(int64_t) * tile_var_sizes__attr#_#1(size_t) * tile_var_sizes_attr#_#2 (size_t) ... * last_tile_cell_num(int64_t) */ int BookKeeping::load(StorageFS *fs) { // Create StorageBuffer to deserialize book_keeping content buffer_ = new CompressedStorageBuffer(fs, filename_, download_compressed_size_, /*is_read*/ true, TILEDB_GZIP, TILEDB_COMPRESSION_LEVEL_GZIP); // Load non-empty domain if(load_non_empty_domain() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Load MBRs if(load_mbrs() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Load bounding coordinates if(load_bounding_coords() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Load tile offsets if(load_tile_offsets() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Load variable tile offsets if(load_tile_var_offsets() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Load variable tile sizes if(load_tile_var_sizes() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Load cell number of last tile if(load_last_tile_cell_num() != TILEDB_BK_OK) return TILEDB_BK_ERR; // Free up StorageBuffer buffer_->finalize(); delete buffer_; buffer_ = 0; // Success return TILEDB_BK_OK; } void BookKeeping::set_last_tile_cell_num(int64_t cell_num) { last_tile_cell_num_ = cell_num; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ /* FORMAT: * bounding_coords_num(int64_t) * bounding_coords_#1(void*) bounding_coords_#2(void*) ... */ int BookKeeping::flush_bounding_coords() { // For easy reference size_t bounding_coords_size = 2*array_schema_->coords_size(); int64_t bounding_coords_num = bounding_coords_.size(); // Write number of bounding coordinates if(buffer_->append_buffer(&bounding_coords_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing number of bounding " "coordinates failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Write bounding coordinates for(int64_t i=0; iappend_buffer(bounding_coords_[i], bounding_coords_size) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing bounding coordinates failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * last_tile_cell_num(int64_t) */ int BookKeeping::flush_last_tile_cell_num() { // For easy reference int64_t cell_num_per_tile = dense_ ? array_schema_->cell_num_per_tile() : array_schema_->capacity(); // Handle the case of zero int64_t last_tile_cell_num = (last_tile_cell_num_ == 0) ? cell_num_per_tile : last_tile_cell_num_; if(buffer_->append_buffer(&last_tile_cell_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing last tile cell number failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Success return TILEDB_BK_OK; } /* FORMAT: * mbr_num(int64_t) * mbr_#1(void*) mbr_#2(void*) ... */ int BookKeeping::flush_mbrs() { // For easy reference size_t mbr_size = 2*array_schema_->coords_size(); int64_t mbr_num = mbrs_.size(); // Write number of MBRs if(buffer_->append_buffer(&mbr_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing number of MBRs failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Write MBRs for(int64_t i=0; iappend_buffer(mbrs_[i], mbr_size) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing MBR failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * non_empty_domain_size(size_t) non_empty_domain(void*) */ int BookKeeping::flush_non_empty_domain() { size_t domain_size = (non_empty_domain_ == NULL) ? 0 : array_schema_->coords_size() * 2; // Write non-empty domain size if (buffer_->append_buffer(&domain_size, sizeof(size_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing domain size failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Write non-empty domain if(non_empty_domain_ != NULL) { if(buffer_->append_buffer(non_empty_domain_, domain_size) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing domain failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * tile_offsets_attr#0_num(int64_t) * tile_offsets_attr#0_#1 (off_t) tile_offsets_attr#0_#2 (off_t) ... * ... * tile_offsets_attr#_num(int64_t) * tile_offsets_attr#_#1 (off_t) * tile_offsets_attr#_#2 (off_t) ... */ int BookKeeping::flush_tile_offsets() { // For easy reference int attribute_num = array_schema_->attribute_num(); int64_t tile_offsets_num; // Write tile offsets for each attribute for(int i=0; iappend_buffer(&tile_offsets_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing number of tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } if(tile_offsets_num == 0) continue; // Write tile offsets if(buffer_->append_buffer(&tile_offsets_[i][0], tile_offsets_num * sizeof(off_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * tile_var_offsets_attr#0_num(int64_t) * tile_var_offsets_attr#0_#1 (off_t) tile_var_offsets_attr#0_#2 (off_t) ... * ... * tile_var_offsets_attr#_num(int64_t) * tile_var_offsets_attr#_#1 (off_t) * tile_var_offsets_attr#_#2 (off_t) ... */ int BookKeeping::flush_tile_var_offsets() { // For easy reference int attribute_num = array_schema_->attribute_num(); int64_t tile_var_offsets_num; // Write tile offsets for each attribute for(int i=0; iappend_buffer(&tile_var_offsets_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing number of " "variable tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } if(tile_var_offsets_num == 0) continue; // Write tile offsets if(buffer_->append_buffer(&tile_var_offsets_[i][0], tile_var_offsets_num * sizeof(off_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing variable tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * tile_var_sizes_attr#0_num(int64_t) * tile_var_sizes_attr#0_#1 (size_t) tile_sizes_attr#0_#2 (size_t) ... * ... * tile_var_sizes_attr#_num(int64_t) * tile_var_sizes__attr#_#1(size_t) * tile_var_sizes_attr#_#2 (size_t) ... */ int BookKeeping::flush_tile_var_sizes() { // For easy reference int attribute_num = array_schema_->attribute_num(); int64_t tile_var_sizes_num; // Write tile sizes for each attribute for(int i=0; iappend_buffer(&tile_var_sizes_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing number of " "variable tile sizes failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } if(tile_var_sizes_num == 0) continue; // Write tile sizes if(buffer_->append_buffer(&tile_var_sizes_[i][0], tile_var_sizes_num * sizeof(size_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot finalize book-keeping; Writing variable tile sizes failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * bounding_coords_num (int64_t) * bounding_coords_#1 (void*) bounding_coords_#2 (void*) ... */ int BookKeeping::load_bounding_coords() { // For easy reference size_t bounding_coords_size = 2*array_schema_->coords_size(); // Get number of bounding coordinates int64_t bounding_coords_num; if(buffer_->read_buffer(&bounding_coords_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading number of " "bounding coordinates failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Get bounding coordinates void* bounding_coords; bounding_coords_.resize(bounding_coords_num); for(int64_t i=0; iread_buffer(bounding_coords, bounding_coords_size) == TILEDB_BF_ERR) { free(bounding_coords); std::string errmsg = "Cannot load book-keeping; Reading bounding coordinates failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } bounding_coords_[i] = bounding_coords; } // Success return TILEDB_BK_OK; } /* FORMAT: * last_tile_cell_num (int64_t) */ int BookKeeping::load_last_tile_cell_num() { // Get last tile cell number if(buffer_->read_buffer(&last_tile_cell_num_, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading last tile cell number failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Success return TILEDB_BK_OK; } /* FORMAT: * mbr_num (int64_t) * mbr_#1 (void*) mbr_#2 (void*) ... mbr_# (void*) */ int BookKeeping::load_mbrs() { // For easy reference size_t mbr_size = 2*array_schema_->coords_size(); // Get number of MBRs int64_t mbr_num; if(buffer_->read_buffer(&mbr_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading number of MBRs failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Get MBRs void* mbr; mbrs_.resize(mbr_num); for(int64_t i=0; iread_buffer(mbr, mbr_size) == TILEDB_BF_ERR) { free(mbr); std::string errmsg = "Cannot load book-keeping; Reading MBR failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } mbrs_[i] = mbr; } // Success return TILEDB_BK_OK; } /* FORMAT: * non_empty_domain_size (size_t) non_empty_domain (void*) */ int BookKeeping::load_non_empty_domain() { // Get domain size size_t domain_size; if(buffer_->read_buffer(&domain_size, sizeof(size_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading domain size failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } // Get non-empty domain if(domain_size == 0) { non_empty_domain_ = NULL; } else { non_empty_domain_ = malloc(domain_size); if(buffer_->read_buffer(non_empty_domain_, domain_size) == TILEDB_BF_ERR) { free(non_empty_domain_); std::string errmsg = "Cannot load book-keeping; Reading domain failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Get expanded domain if(non_empty_domain_ == NULL) { domain_ = NULL; } else { domain_ = malloc(domain_size); memcpy(domain_, non_empty_domain_, domain_size); array_schema_->expand_domain(domain_); } // Success return TILEDB_BK_OK; } /* FORMAT: * tile_offsets_attr#0_num (int64_t) * tile_offsets_attr#0_#1 (off_t) tile_offsets_attr#0_#2 (off_t) ... * ... * tile_offsets_attr#_num (int64_t) * tile_offsets_attr#_#1 (off_t) * tile_offsets_attr#_#2 (off_t) ... */ int BookKeeping::load_tile_offsets() { // For easy reference int attribute_num = array_schema_->attribute_num(); int64_t tile_offsets_num; // Allocate tile offsets tile_offsets_.resize(attribute_num+1); // For all attributes, get the tile offsets for(int i=0; iread_buffer(&tile_offsets_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading number of tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } if(tile_offsets_num == 0) continue; // Get tile offsets tile_offsets_[i].resize(tile_offsets_num); if(buffer_->read_buffer(&tile_offsets_[i][0], tile_offsets_num * sizeof(off_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * tile_var_offsets_attr#0_num (int64_t) * tile_var_offsets_attr#0_#1 (off_t) tile_var_offsets_attr#0_#2 (off_t) ... * ... * tile_var_offsets_attr#_num(int64_t) * tile_var_offsets_attr#_#1 (off_t) * tile_ver_offsets_attr#_#2 (off_t) ... */ int BookKeeping::load_tile_var_offsets() { // For easy reference int attribute_num = array_schema_->attribute_num(); int64_t tile_var_offsets_num; // Allocate tile offsets tile_var_offsets_.resize(attribute_num); // For all attributes, get the variable tile offsets for(int i=0; iread_buffer(&tile_var_offsets_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading number of variable tile " "offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } if(tile_var_offsets_num == 0) continue; // Get variable tile offsets tile_var_offsets_[i].resize(tile_var_offsets_num); if(buffer_->read_buffer(&tile_var_offsets_[i][0], tile_var_offsets_num * sizeof(off_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading variable tile offsets failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } /* FORMAT: * tile_var_sizes_attr#0_num (int64_t) * tile_var_sizes_attr#0_#1 (size_t) tile_sizes_attr#0_#2 (size_t) ... * ... * tile_var_sizes_attr#_num( int64_t) * tile_var_sizes__attr#_#1 (size_t) * tile_var_sizes_attr#_#2 (size_t) ... */ int BookKeeping::load_tile_var_sizes() { // For easy reference int attribute_num = array_schema_->attribute_num(); int64_t tile_var_sizes_num; // Allocate tile sizes tile_var_sizes_.resize(attribute_num); // For all attributes, get the variable tile sizes for(int i=0; iread_buffer(&tile_var_sizes_num, sizeof(int64_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading number of variable tile " "sizes failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } if(tile_var_sizes_num == 0) continue; // Get variable tile sizes tile_var_sizes_[i].resize(tile_var_sizes_num); if(buffer_->read_buffer(&tile_var_sizes_[i][0], tile_var_sizes_num * sizeof(size_t)) == TILEDB_BF_ERR) { std::string errmsg = "Cannot load book-keeping; Reading variable tile sizes failed"; PRINT_ERROR(errmsg); tiledb_bk_errmsg = TILEDB_BK_ERRMSG + errmsg; return TILEDB_BK_ERR; } } // Success return TILEDB_BK_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/fragment/fragment.cc000066400000000000000000000203061453617025200240020ustar00rootroot00000000000000/** * @file fragment.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2019, 2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the Fragment class. */ #include "fragment.h" #include "tiledb_constants.h" #include "utils.h" #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_FG_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_fg_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ Fragment::Fragment(const Array* array) : array_(array) { read_state_ = NULL; write_state_ = NULL; book_keeping_ = NULL; } Fragment::~Fragment() { if(write_state_ != NULL) delete write_state_; if(read_state_ != NULL) delete read_state_; if(book_keeping_ != NULL && !read_mode()) delete book_keeping_; } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ const Array* Fragment::array() const { return array_; } int64_t Fragment::cell_num_per_tile() const { return (dense_) ? array_->array_schema()->cell_num_per_tile() : array_->array_schema()->capacity(); } bool Fragment::dense() const { return dense_; } const std::string& Fragment::fragment_name() const { return fragment_name_; } int Fragment::mode() const { return mode_; } BookKeeping* Fragment::book_keeping() const { return book_keeping_; } inline bool Fragment::read_mode() const { return array_read_mode(mode_); } ReadState* Fragment::read_state() const { return read_state_; } size_t Fragment::tile_size(int attribute_id) const { // For easy reference const ArraySchema* array_schema = array_->array_schema(); bool var_size = array_schema->var_size(attribute_id); int64_t cell_num_per_tile = (dense_) ? array_schema->cell_num_per_tile() : array_schema->capacity(); return (var_size) ? cell_num_per_tile * TILEDB_CELL_VAR_OFFSET_SIZE : cell_num_per_tile * array_schema->cell_size(attribute_id); } inline bool Fragment::write_mode() const { return array_write_mode(mode_); } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int Fragment::finalize() { if(write_state_ != NULL) { // WRITE StorageFS *fs = array_->config()->get_filesystem(); assert(book_keeping_ != NULL); int rc_ws = write_state_->finalize(); int rc_bk = book_keeping_->finalize(fs); // Errors if(rc_ws != TILEDB_WS_OK) { tiledb_fg_errmsg = tiledb_ws_errmsg; return TILEDB_FG_ERR; } if(rc_bk != TILEDB_BK_OK) { tiledb_fg_errmsg = tiledb_bk_errmsg; return TILEDB_FG_ERR; } if(is_dir(fs, fragment_name_)) { if (rename_fragment()) { return TILEDB_FG_ERR; } if (create_fragment_file(fs, fragment_name_)) { tiledb_fg_errmsg = tiledb_ut_errmsg; return TILEDB_FG_ERR; } } // Success return TILEDB_FG_OK; } else { // READ return read_state_->finalize(); } } int Fragment::init( const std::string& fragment_name, int mode, const void* subarray) { // Set fragment name and mode fragment_name_ = fragment_name; mode_ = mode; // Sanity check if(!write_mode()) { std::string errmsg = "Cannot initialize fragment; Invalid mode"; PRINT_ERROR(errmsg); tiledb_fg_errmsg = TILEDB_FG_ERRMSG + errmsg; return TILEDB_FG_ERR; } // Check if the fragment is dense or not dense_ = true; const std::vector& attribute_ids = array_->attribute_ids(); int id_num = attribute_ids.size(); int attribute_num = array_->array_schema()->attribute_num(); for(int i=0; iarray_schema(), dense_, fragment_name, mode_); read_state_ = NULL; if(book_keeping_->init(subarray) != TILEDB_BK_OK) { delete book_keeping_; book_keeping_ = NULL; write_state_ = NULL; tiledb_fg_errmsg = tiledb_bk_errmsg; return TILEDB_FG_ERR; } write_state_ = new WriteState(this, book_keeping_); // Success return TILEDB_FG_OK; } int Fragment::init( const std::string& fragment_name, BookKeeping* book_keeping, int mode) { // Set member attributes fragment_name_ = fragment_name; mode_ = mode; // Sanity check if(!read_mode()) { std::string errmsg = "Cannot initialize fragment; Invalid mode"; PRINT_ERROR(errmsg); tiledb_fg_errmsg = TILEDB_FG_ERRMSG + errmsg; return TILEDB_FG_ERR; } // Set book_keeping and initialize read/write state book_keeping_ = book_keeping; dense_ = book_keeping_->dense(); write_state_ = NULL; read_state_ = new ReadState(this, book_keeping_); // Success return TILEDB_FG_OK; } void Fragment::reset_read_state() { read_state_->reset(); } int Fragment::sync() { // Sanity check assert(write_state_ != NULL); // Sync if(write_state_->sync() != TILEDB_WS_OK) { tiledb_fg_errmsg = tiledb_ws_errmsg; return TILEDB_FG_ERR; } else { return TILEDB_FG_OK; } } int Fragment::sync_attribute(const std::string& attribute) { // Sanity check assert(write_state_ != NULL); // Sync attribute if(write_state_->sync_attribute(attribute) != TILEDB_WS_OK) { tiledb_fg_errmsg = tiledb_ws_errmsg; return TILEDB_FG_ERR; } else { return TILEDB_FG_OK; } } int Fragment::write(const void** buffers, const size_t* buffer_sizes) { // Forward the write command to the write state int rc = write_state_->write(buffers, buffer_sizes); // Error if(rc != TILEDB_WS_OK) { tiledb_fg_errmsg = tiledb_ws_errmsg; return TILEDB_FG_ERR; } // Success return TILEDB_FG_OK; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ int Fragment::rename_fragment() { // Do nothing in READ mode if(read_mode()) return TILEDB_FG_OK; StorageFS *fs = array_->config()->get_filesystem(); // No rename of fragment for filesystems with no locking if (!fs->locking_support()) { return TILEDB_FG_OK; } std::string parent_dir = ::parent_dir(fs, fragment_name_); std::string new_fragment_name = parent_dir + "/" + ::real_dir(fs, fragment_name_).substr(parent_dir.size() + 2); if(move_path(fs, fragment_name_, new_fragment_name) == TILEDB_UT_ERR) { std::string errmsg = std::string("Cannot rename fragment directory"); PRINT_ERROR(errmsg); tiledb_fg_errmsg = TILEDB_FG_ERRMSG + errmsg; return TILEDB_FG_ERR; } fragment_name_ = new_fragment_name; return TILEDB_FG_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/fragment/read_state.cc000066400000000000000000002564211453617025200243230ustar00rootroot00000000000000/** * @file read_state.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the ReadState class. */ #include "read_state.h" #include "utils.h" #include #include #include #include #include #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_RS_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_rs_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ReadState::ReadState( const Fragment* fragment, BookKeeping* book_keeping) : book_keeping_(book_keeping), fragment_(fragment) { array_ = fragment_->array(); array_schema_ = array_->array_schema(); attribute_num_ = array_schema_->attribute_num(); coords_size_ = array_schema_->coords_size(); done_ = false; fetched_tile_.resize(attribute_num_+2); overflow_.resize(attribute_num_+1); last_tile_coords_ = NULL; map_addr_.resize(attribute_num_+2); map_addr_lengths_.resize(attribute_num_+2); map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; map_addr_var_.resize(attribute_num_); map_addr_var_lengths_.resize(attribute_num_); search_tile_overlap_subarray_ = malloc(2*coords_size_); search_tile_pos_ = -1; tile_compressed_ = NULL; tile_compressed_allocated_size_ = 0; tiles_.resize(attribute_num_+2); tiles_offsets_.resize(attribute_num_+2); tiles_file_offsets_.resize(attribute_num_+2); tiles_sizes_.resize(attribute_num_+2); tiles_var_.resize(attribute_num_); tiles_var_offsets_.resize(attribute_num_); tiles_var_file_offsets_.resize(attribute_num_); tiles_var_sizes_.resize(attribute_num_); tiles_var_allocated_size_.resize(attribute_num_); tmp_coords_ = malloc(coords_size_); for(int i=0; iconfig()->get_filesystem(); for(int i=0; ifile_size(construct_filename(i)); file_var_size_[i] = fs->file_size(construct_filename(i, /*is_var*/true)); } // Setup file buffers for buffered reading per attribute+coords file_buffer_.resize(attribute_num_+1); file_var_buffer_.resize(attribute_num_+1); reset_file_buffers(); // Get compression for tiles per attribute+coords+search_tile from schema codec_.resize(attribute_num_+2); for(int i=0; ivar_size(i)) { offsets_codec_[i] = Codec::create(array_schema_, i, true); } else { offsets_codec_[i] = NULL; } } } ReadState::~ReadState() { // Delete codec instances for(auto i=0u; idense(); } bool ReadState::done() const { return done_; } void ReadState::get_bounding_coords(void* bounding_coords) const { // For easy reference int64_t pos = search_tile_pos_; assert(pos != -1); memcpy( bounding_coords, book_keeping_->bounding_coords()[pos], 2*coords_size_); } bool ReadState::mbr_overlaps_tile() const { return mbr_tile_overlap_; } bool ReadState::overflow(int attribute_id) const { return overflow_[attribute_id]; } bool ReadState::subarray_area_covered() const { return subarray_area_covered_; } /* ****************************** */ /* MUTATORS */ /* ****************************** */ void ReadState::reset() { reset_file_buffers(); if(last_tile_coords_ != NULL) { free(last_tile_coords_); last_tile_coords_ = NULL; } reset_overflow(); done_ = false; search_tile_pos_ = -1; compute_tile_search_range(); for(int i=0; icell_size(attribute_id); //If this tile hasn't been fetched and all the cells in this tile should be skipped, //then there is no need to fetch the tile from disk (and decompress) size_t num_cells_in_curr_tile = (cell_pos_range.second - cell_pos_range.first + 1u); if(fetched_tile_[attribute_id] != tile_i && remaining_skip_count >= num_cells_in_curr_tile) { remaining_skip_count -= num_cells_in_curr_tile; return TILEDB_RS_OK; } // Calculate free space in buffer size_t buffer_free_space = buffer_size - buffer_offset; buffer_free_space = (buffer_free_space / cell_size) * cell_size; if(buffer_free_space == 0 && remaining_skip_count == 0u) { // Overflow overflow_[attribute_id] = true; return TILEDB_RS_OK; } // Prepare attribute tile if(prepare_tile_for_reading(attribute_id, tile_i) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Sanity check assert(!array_schema_->var_size(attribute_id)); // For each cell position range, copy the respective cells to the buffer size_t start_offset, end_offset; size_t bytes_left_to_copy, bytes_to_copy; // Calculate start and end offset in the tile start_offset = cell_pos_range.first * cell_size; end_offset = (cell_pos_range.second + 1) * cell_size - 1; // Potentially set the tile offset to the beginning of the current range if(tiles_offsets_[attribute_id] < start_offset) tiles_offsets_[attribute_id] = start_offset; else if(tiles_offsets_[attribute_id] > end_offset) // This range is written return TILEDB_RS_OK; // Calculate number of bytes to skip auto bytes_to_skip = remaining_skip_count * cell_size; //#cells remaining in this cell range <= remaining_skip_count if(tiles_offsets_[attribute_id] + bytes_to_skip > end_offset) { // This range is written assert(remaining_skip_count > 0u); auto num_cells_skipped = (end_offset - tiles_offsets_[attribute_id] + 1u)/cell_size; assert(num_cells_skipped <= remaining_skip_count); remaining_skip_count -= num_cells_skipped; return TILEDB_RS_OK; } //skip some cells tiles_offsets_[attribute_id] += bytes_to_skip; // Calculate the total size to copy bytes_left_to_copy = end_offset - tiles_offsets_[attribute_id] + 1; bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); // Copy and update current buffer and tile offsets if(bytes_to_copy != 0) { if(READ_FROM_TILE( attribute_id, static_cast(buffer) + buffer_offset, tiles_offsets_[attribute_id], bytes_to_copy) != TILEDB_RS_OK) return TILEDB_RS_ERR; buffer_offset += bytes_to_copy; tiles_offsets_[attribute_id] += bytes_to_copy; buffer_free_space = buffer_size - buffer_offset; } //if some cells had remained to be skipped, they would have been caught by the if statement 30 lines above remaining_skip_count = 0u; // Handle buffer overflow if(tiles_offsets_[attribute_id] != end_offset + 1) overflow_[attribute_id] = true; // Success return TILEDB_RS_OK; } int ReadState::copy_cells_var( int attribute_id, int tile_i, void* buffer, size_t buffer_size, size_t& buffer_offset, size_t& remaining_skip_count, void* buffer_var, size_t buffer_var_size, size_t& buffer_var_offset, size_t& remaining_skip_count_var, const CellPosRange& cell_pos_range) { // TileDB traverses the offsets and data in lock step - can't have different skip values assert(remaining_skip_count == remaining_skip_count_var); //If this tile hasn't been fetched and all the cells in this tile should be skipped, //then there is no need to fetch the tile from disk (and decompress) size_t num_cells_in_curr_tile = (cell_pos_range.second - cell_pos_range.first + 1u); if(fetched_tile_[attribute_id] != tile_i && remaining_skip_count >= num_cells_in_curr_tile) { remaining_skip_count -= num_cells_in_curr_tile; remaining_skip_count_var -= num_cells_in_curr_tile; return TILEDB_RS_OK; } // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; // Calculate free space in buffer size_t buffer_free_space = buffer_size - buffer_offset; buffer_free_space = (buffer_free_space / cell_size) * cell_size; size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; // Handle overflow if((buffer_free_space == 0 || buffer_var_free_space == 0) && remaining_skip_count == 0u) { // Overflow overflow_[attribute_id] = true; return TILEDB_RS_OK; } // Prepare attribute tile if(prepare_tile_for_reading_var(attribute_id, tile_i) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Sanity check assert(array_schema_->var_size(attribute_id)); // For each cell position range, copy the respective cells to the buffer size_t start_offset, end_offset; int64_t start_cell_pos, end_cell_pos; size_t bytes_left_to_copy, bytes_to_copy, bytes_var_to_copy; // Calculate start and end offset in the tile start_offset = cell_pos_range.first * cell_size; end_offset = (cell_pos_range.second + 1) * cell_size - 1; // Potentially set the tile offset to the beginning of the current range if(tiles_offsets_[attribute_id] < start_offset) tiles_offsets_[attribute_id] = start_offset; else if(tiles_offsets_[attribute_id] > end_offset) // This range is written return TILEDB_RS_OK; // Calculate number of bytes to skip auto bytes_to_skip = remaining_skip_count * cell_size; //#cells remaining in this cell range <= remaining_skip_count if(tiles_offsets_[attribute_id] + bytes_to_skip > end_offset) { // This range is written assert(remaining_skip_count > 0u); auto num_cells_skipped = (end_offset - tiles_offsets_[attribute_id] + 1u)/cell_size; assert(num_cells_skipped <= remaining_skip_count); remaining_skip_count -= num_cells_skipped; remaining_skip_count_var -= num_cells_skipped; return TILEDB_RS_OK; } //skip some cells tiles_offsets_[attribute_id] += bytes_to_skip; // Calculate the total size to copy bytes_left_to_copy = end_offset - tiles_offsets_[attribute_id] + 1; bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); // Compute actual bytes to copy start_cell_pos = tiles_offsets_[attribute_id] / cell_size; end_cell_pos = start_cell_pos + bytes_to_copy/cell_size - 1; if(compute_bytes_to_copy( attribute_id, start_cell_pos, end_cell_pos, buffer_free_space, buffer_var_free_space, bytes_to_copy, bytes_var_to_copy) != TILEDB_RS_OK) return TILEDB_RS_ERR; // For easy reference void* buffer_start = static_cast(buffer) + buffer_offset; // Potentially update tile offset to the beginning of the overlap range const size_t* tile_var_start; if(GET_CELL_PTR_FROM_OFFSET_TILE( attribute_id, start_cell_pos, tile_var_start) != TILEDB_RS_OK) return TILEDB_RS_ERR; // this also takes care of skipped cells if(tiles_var_offsets_[attribute_id] < *tile_var_start) tiles_var_offsets_[attribute_id] = *tile_var_start; // Copy and update current buffer and tile offsets if(bytes_to_copy != 0) { if(READ_FROM_TILE( attribute_id, buffer_start, tiles_offsets_[attribute_id], bytes_to_copy) != TILEDB_RS_OK) return TILEDB_RS_ERR; buffer_offset += bytes_to_copy; tiles_offsets_[attribute_id] += bytes_to_copy; buffer_free_space = buffer_size - buffer_offset; // Shift variable offsets shift_var_offsets( buffer_start, end_cell_pos - start_cell_pos + 1, buffer_var_offset); // Copy and update current variable buffer and tile offsets if(READ_FROM_TILE_VAR( attribute_id, static_cast(buffer_var) + buffer_var_offset, tiles_var_offsets_[attribute_id], bytes_var_to_copy) != TILEDB_RS_OK) return TILEDB_RS_ERR; buffer_var_offset += bytes_var_to_copy; tiles_var_offsets_[attribute_id] += bytes_var_to_copy; buffer_var_free_space = buffer_var_size - buffer_var_offset; } // Check for overflow if(tiles_offsets_[attribute_id] != end_offset + 1) overflow_[attribute_id] = true; //if some cells had remained to be skipped, they would have been caught by the if statement 80 lines above remaining_skip_count = 0u; remaining_skip_count_var = 0u; // Entering this if condition implies that the var data in this cell is so // large that the allocated buffer cannot hold it if(buffer_offset == 0u && bytes_to_copy == 0u) { overflow_[attribute_id] = true; return TILEDB_RS_OK; } // Success return TILEDB_RS_OK; } template int ReadState::get_coords_after( const T* coords, T* coords_after, bool& coords_retrieved) { // For easy reference int64_t cell_num = book_keeping_->cell_num(search_tile_pos_); // Prepare attribute tile if(prepare_tile_for_reading(attribute_num_+1, search_tile_pos_) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Compute the cell position at or after the coords int64_t coords_after_pos = get_cell_pos_after(coords); // Invalid position if(coords_after_pos < 0 || coords_after_pos >= cell_num) { coords_retrieved = false; return TILEDB_RS_OK; } // Copy result if(READ_FROM_TILE( attribute_num_+1, coords_after, coords_after_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; coords_retrieved = true; // Success return TILEDB_RS_OK; } template int ReadState::get_enclosing_coords( int tile_i, const T* target_coords, const T* start_coords, const T* end_coords, T* left_coords, T* right_coords, bool& left_retrieved, bool& right_retrieved, bool& target_exists) { // Prepare attribute tile if(prepare_tile_for_reading(attribute_num_+1, tile_i) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Compute the appropriate cell positions int64_t start_pos = get_cell_pos_at_or_after(start_coords); int64_t end_pos = get_cell_pos_at_or_before(end_coords); int64_t target_pos = get_cell_pos_at_or_before(target_coords); // Check if target exists if(target_pos >= start_pos && target_pos <= end_pos) { int cmp = CMP_COORDS_TO_SEARCH_TILE( target_coords, target_pos*coords_size_); if(cmp == TILEDB_RS_ERR) return TILEDB_RS_ERR; if(cmp) target_exists = true; else target_exists = false; } else { target_exists = false; } // Calculate left and right pos int64_t left_pos = (target_exists) ? target_pos-1 : target_pos; int64_t right_pos = target_pos+1; // Copy left if it exists if(left_pos >= start_pos && left_pos <= end_pos) { if(READ_FROM_TILE( attribute_num_+1, left_coords, left_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; left_retrieved = true; } else { left_retrieved = false; } // Copy right if it exists if(right_pos >= start_pos && right_pos <= end_pos) { if(READ_FROM_TILE( attribute_num_+1, right_coords, right_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; right_retrieved = true; } else { right_retrieved = false; } // Success return TILEDB_RS_OK; } template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const T* cell_range, FragmentCellPosRange& fragment_cell_pos_range) { // For easy reference int dim_num = array_schema_->dim_num(); int64_t tile_i = fragment_info.second; // Prepare attribute tile if(prepare_tile_for_reading(attribute_num_+1, tile_i) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Compute the appropriate cell positions int64_t start_pos = get_cell_pos_at_or_after(cell_range); int64_t end_pos = get_cell_pos_at_or_before(&cell_range[dim_num]); // Create the result fragment_cell_pos_range.first = fragment_info; if(start_pos <= end_pos) // There are results fragment_cell_pos_range.second = CellPosRange(start_pos, end_pos); else // There are NO rsults fragment_cell_pos_range.second = CellPosRange(-1, -1); // Success return TILEDB_RS_OK; } template int ReadState::get_fragment_cell_ranges_dense( int fragment_i, FragmentCellRanges& fragment_cell_ranges) { // Trivial cases if(done_ || !search_tile_overlap_) return TILEDB_RS_OK; // For easy reference int dim_num = array_schema_->dim_num(); int cell_order = array_schema_->cell_order(); size_t cell_range_size = 2*coords_size_; const T* search_tile_overlap_subarray = static_cast(search_tile_overlap_subarray_); FragmentInfo fragment_info = FragmentInfo(fragment_i, search_tile_pos_); // Contiguous cells, single cell range if(search_tile_overlap_ == 1 || search_tile_overlap_ == 3) { void* cell_range = malloc(cell_range_size); T* cell_range_T = static_cast(cell_range); for(int i=0; i(cell_range); for(int i=0; i 0 && coords[i] > search_tile_overlap_subarray[2*i+1]) { coords[i] = search_tile_overlap_subarray[2*i]; ++coords[--i]; } } } else if(cell_order == TILEDB_COL_MAJOR) { // COLUMN while(coords[dim_num-1] <= search_tile_overlap_subarray[2*(dim_num-1)+1]) { // Make a cell range representing a slab void* cell_range = malloc(cell_range_size); T* cell_range_T = static_cast(cell_range); for(int i=dim_num-1; i>0; --i) { cell_range_T[i] = coords[i]; cell_range_T[dim_num+i] = coords[i]; } cell_range_T[0] = search_tile_overlap_subarray[0]; cell_range_T[dim_num] = search_tile_overlap_subarray[1]; // Insert the new range into the result vector fragment_cell_ranges.push_back( FragmentCellRange(fragment_info, cell_range)); // Advance coordinates i=1; ++coords[i]; while(i search_tile_overlap_subarray[2*i+1]) { coords[i] = search_tile_overlap_subarray[2*i]; ++coords[++i]; } } } else { assert(0); } // Clean up delete [] coords; } // Success return TILEDB_RS_OK; } template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, FragmentCellRanges& fragment_cell_ranges) { // Trivial cases if(done_ || !search_tile_overlap_ || !mbr_tile_overlap_) return TILEDB_RS_OK; // For easy reference int dim_num = array_schema_->dim_num(); const T* search_tile_overlap_subarray = static_cast(search_tile_overlap_subarray_); // Create start and end coordinates for the overlap T* start_coords = new T[dim_num]; T* end_coords = new T[dim_num]; for(int i=0; i int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, const T* start_coords, const T* end_coords, FragmentCellRanges& fragment_cell_ranges) { // Sanity checks assert(search_tile_pos_ >= tile_search_range_[0] && search_tile_pos_ <= tile_search_range_[1]); assert(search_tile_overlap_); // For easy reference int dim_num = array_schema_->dim_num(); const T* subarray = static_cast(array_->subarray()); // Handle full overlap if(search_tile_overlap_ == 1) { FragmentCellRange fragment_cell_range; fragment_cell_range.first = FragmentInfo(fragment_i, search_tile_pos_); fragment_cell_range.second = malloc(2*coords_size_); T* cell_range = static_cast(fragment_cell_range.second); memcpy(cell_range, start_coords, coords_size_); memcpy(&cell_range[dim_num], end_coords, coords_size_); fragment_cell_ranges.push_back(fragment_cell_range); return TILEDB_RS_OK; } // Prepare attribute tile if(prepare_tile_for_reading(attribute_num_+1, search_tile_pos_) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Get cell positions for the cell range int64_t start_pos = get_cell_pos_at_or_after(start_coords); int64_t end_pos = get_cell_pos_at_or_before(end_coords); // Get the cell ranges const void* cell; int64_t current_start_pos = 0, current_end_pos = -2; for(int64_t i=start_pos; i<=end_pos; ++i) { if(GET_COORDS_PTR_FROM_SEARCH_TILE(i, cell) != TILEDB_RS_OK) return TILEDB_RS_ERR; if(cell_in_subarray(static_cast(cell), subarray, dim_num)) { if(i-1 == current_end_pos) { // The range is expanded ++current_end_pos; } else { // A new range starts current_start_pos = i; current_end_pos = i; } } else { if(i-1 == current_end_pos) { // The range needs to be added to the list FragmentCellRange fragment_cell_range; fragment_cell_range.first = FragmentInfo(fragment_i, search_tile_pos_); fragment_cell_range.second = malloc(2*coords_size_); T* cell_range = static_cast(fragment_cell_range.second); if(READ_FROM_TILE( attribute_num_+1, cell_range, current_start_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; if(READ_FROM_TILE( attribute_num_+1, &cell_range[dim_num], current_end_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; fragment_cell_ranges.push_back(fragment_cell_range); current_end_pos = -2; // This indicates that there is no active range } } } // Add last cell range if(current_end_pos != -2) { FragmentCellRange fragment_cell_range; fragment_cell_range.first = FragmentInfo(fragment_i, search_tile_pos_); fragment_cell_range.second = malloc(2*coords_size_); T* cell_range = static_cast(fragment_cell_range.second); if(READ_FROM_TILE( attribute_num_+1, cell_range, current_start_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; if(READ_FROM_TILE( attribute_num_+1, &cell_range[dim_num], current_end_pos*coords_size_, coords_size_) != TILEDB_RS_OK) return TILEDB_RS_ERR; fragment_cell_ranges.push_back(fragment_cell_range); } // Success return TILEDB_RS_OK; } template void ReadState::get_next_overlapping_tile_dense(const T* tile_coords) { // Trivial case if(done_) return; // For easy reference int dim_num = array_schema_->dim_num(); const T* tile_extents = static_cast(array_schema_->tile_extents()); const T* array_domain = static_cast(array_schema_->domain()); const T* subarray = static_cast(array_->subarray()); const T* domain = static_cast(book_keeping_->domain()); const T* non_empty_domain = static_cast(book_keeping_->non_empty_domain()); // Compute the tile subarray T* tile_subarray = new T[2*dim_num]; array_schema_->get_tile_subarray(tile_coords, tile_subarray); // Compute overlap of tile subarray with non-empty fragment domain T* tile_domain_overlap_subarray = new T[2*dim_num]; bool tile_domain_overlap = array_schema_->subarray_overlap( tile_subarray, non_empty_domain, tile_domain_overlap_subarray); if(!tile_domain_overlap) { // No overlap with the input tile search_tile_overlap_ = 0; subarray_area_covered_ = false; } else { // Overlap with the input tile // Find the search tile position T* tile_coords_norm = new T[dim_num]; for(int i=0; iget_tile_pos(domain, tile_coords_norm); delete [] tile_coords_norm; // Compute overlap of the query subarray with tile T* query_tile_overlap_subarray = new T[2*dim_num]; array_schema_->subarray_overlap( subarray, tile_subarray, query_tile_overlap_subarray); // Compute the overlap of the previous results with the non-empty domain T* search_tile_overlap_subarray = (T*) search_tile_overlap_subarray_; bool overlap = array_schema_->subarray_overlap( query_tile_overlap_subarray, tile_domain_overlap_subarray, search_tile_overlap_subarray); if(!overlap) { search_tile_overlap_ = 0; subarray_area_covered_ = false; } else { // Find the type of the search tile overlap T* temp = new T[2*dim_num]; search_tile_overlap_ = array_schema_->subarray_overlap( search_tile_overlap_subarray, tile_subarray, temp); // Check if fragment fully covers the tile subarray_area_covered_ = is_contained( query_tile_overlap_subarray, tile_domain_overlap_subarray, dim_num); // Clean up delete [] temp; } // Clean up delete [] query_tile_overlap_subarray; } // Clean up delete [] tile_subarray; delete [] tile_domain_overlap_subarray; } template void ReadState::get_next_overlapping_tile_sparse() { // Trivial case if(done_) return; // For easy reference const std::vector& mbrs = book_keeping_->mbrs(); const T* subarray = static_cast(array_->subarray()); // Update the search tile position if(search_tile_pos_ == -1) search_tile_pos_ = tile_search_range_[0]; else ++search_tile_pos_; // Find the position to the next overlapping tile with the query range for(;;) { // No overlap - exit if(search_tile_pos_ > tile_search_range_[1]) { done_ = true; return; } const T* mbr = static_cast(mbrs[search_tile_pos_]); search_tile_overlap_ = array_schema_->subarray_overlap( subarray, mbr, static_cast(search_tile_overlap_subarray_)); if(!search_tile_overlap_) ++search_tile_pos_; else return; } } template void ReadState::get_next_overlapping_tile_sparse( const T* tile_coords) { // Trivial case if(done_) return; // For easy reference int dim_num = array_schema_->dim_num(); const std::vector& mbrs = book_keeping_->mbrs(); const T* subarray = static_cast(array_->subarray()); // Compute the tile subarray T* tile_subarray = new T[2*dim_num]; T* mbr_tile_overlap_subarray = new T[2*dim_num]; T* tile_subarray_end = new T[dim_num]; array_schema_->get_tile_subarray(tile_coords, tile_subarray); for(int i=0; i( book_keeping_->bounding_coords()[search_tile_pos_]); if(array_schema_->tile_cell_order_cmp( &bounding_coords[dim_num], tile_subarray_end) <= 0) { ++search_tile_pos_; } else { return; } } else { memcpy(last_tile_coords_, tile_coords, coords_size_); } } // Find the position to the next overlapping tile with the input tile for(;;) { // No overlap - exit if(search_tile_pos_ > tile_search_range_[1]) { done_ = true; break; } // Get overlap between MBR and tile subarray const T* mbr = static_cast(mbrs[search_tile_pos_]); mbr_tile_overlap_ = array_schema_->subarray_overlap( tile_subarray, mbr, mbr_tile_overlap_subarray); // No overlap with the tile if(!mbr_tile_overlap_) { // Check if we need to break or continue const T* bounding_coords = static_cast( book_keeping_->bounding_coords()[search_tile_pos_]); if(array_schema_->tile_cell_order_cmp( &bounding_coords[dim_num], tile_subarray_end) > 0) { break; } else { ++search_tile_pos_; continue; } } // Get overlap of MBR with the query inside the tile subarray search_tile_overlap_ = array_schema_->subarray_overlap( subarray, mbr_tile_overlap_subarray, static_cast(search_tile_overlap_subarray_)); // Update the search tile overlap if necessary if(search_tile_overlap_) { // The overlap is full only when both the MBR-tile and MBR-tile-subarray // overlaps are full search_tile_overlap_ = (mbr_tile_overlap_ == 1 && search_tile_overlap_ == 1) ? 1 : 2; } // The MBR overlaps with the tile. Regardless of overlap with the // query in the tile, break. break; } // Clean up delete [] tile_subarray; delete [] tile_subarray_end; delete [] mbr_tile_overlap_subarray; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ std::string ReadState::construct_filename(int attribute_id, bool is_var) { std::string filename; if (attribute_id == attribute_num_) { filename = fragment_->fragment_name() + "/" + TILEDB_COORDS + TILEDB_FILE_SUFFIX; } else { filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id) + (is_var?"_var":"") +TILEDB_FILE_SUFFIX; } return filename; } void ReadState::reset_file_buffers() { for(int i=0; iconfig()->get_filesystem(); close_file(fs, construct_filename(i, true)); close_file(fs, construct_filename(i, false)); } } int ReadState::read_segment(int attribute_num, bool is_var, off_t offset, void *segment, size_t length) { int rc = TILEDB_RS_OK; StorageFS *fs = array_->config()->get_filesystem(); // Special case for coords if (attribute_num==attribute_num_+1) { attribute_num = attribute_num_; } // Construct the attribute file name std::string filename = construct_filename(attribute_num, is_var); // Buffered reading to help with distributed filesystem and cloud performance if (fs->get_download_buffer_size() > 0) { StorageBuffer *file_buffer; if (is_var) { assert((attribute_num < attribute_num_) && "Coords attribute cannot be variable"); if (file_var_buffer_[attribute_num] == NULL) { file_var_buffer_[attribute_num]= new StorageBuffer(fs, filename, fs->get_download_buffer_size(), true); } file_buffer = file_var_buffer_[attribute_num]; } else { if (file_buffer_[attribute_num] == NULL) { file_buffer_[attribute_num] = new StorageBuffer(fs, filename, fs->get_download_buffer_size(), true); } file_buffer = file_buffer_[attribute_num]; } // Read from file buffers if possible if (file_buffer != NULL) { if (file_buffer->read_buffer(offset, segment, length) == TILEDB_BF_ERR) { std::string errmsg = "Cannot read attribute file " + filename + " from memory. Will try read directly from file"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; } else { return rc; } } } // Read segment directly int read_method = array_->config()->read_method(); #ifdef HAVE_MPI MPI_Comm* mpi_comm = array_->config()->mpi_comm(); #endif if(read_method == TILEDB_IO_READ || read_method == TILEDB_IO_MMAP) { rc = read_from_file(fs, filename, offset, segment, length); } else if(read_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_read_from_file(mpi_comm, filename, offset, segment, length); #else // Error: MPI not supported std::string errmsg = "Cannot read MPI file as MPI is not supported"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; #endif } if (rc == TILEDB_UT_ERR) { std::string errmsg = "Cannot read segment from attribute file " + filename; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; } return rc; } int ReadState::CMP_COORDS_TO_SEARCH_TILE( const void* buffer, size_t tile_offset) { // For easy reference char* tile = static_cast(tiles_[attribute_num_+1]); // The tile is in main memory if(tile != NULL) { return !memcmp(buffer, tile + tile_offset, coords_size_); } // Read from file if (read_segment(attribute_num_, false, tiles_file_offsets_[attribute_num_+1]+tile_offset, tmp_coords_, coords_size_) == TILEDB_RS_ERR) { return TILEDB_RS_ERR; } // Return return !memcmp(buffer, tmp_coords_, coords_size_); } int ReadState::compute_bytes_to_copy( int attribute_id, int64_t start_cell_pos, int64_t& end_cell_pos, size_t buffer_free_space, size_t buffer_var_free_space, size_t& bytes_to_copy, size_t& bytes_var_to_copy) { // Trivial case if(buffer_free_space == 0 || buffer_var_free_space == 0) { bytes_to_copy = 0; bytes_var_to_copy = 0; return TILEDB_RS_OK; } // Calculate number of cells in the current tile for this attribute int64_t cell_num = book_keeping_->cell_num(fetched_tile_[attribute_id]); // TODO: Refine this algorithm! // Calculate bytes to copy from the variable tile const size_t* start_offset; const size_t* end_offset; const size_t* med_offset; if(GET_CELL_PTR_FROM_OFFSET_TILE( attribute_id, start_cell_pos, start_offset) != TILEDB_RS_OK) return TILEDB_RS_ERR; if(end_cell_pos + 1 < cell_num) { if(GET_CELL_PTR_FROM_OFFSET_TILE( attribute_id, end_cell_pos+1, end_offset) != TILEDB_RS_OK) return TILEDB_RS_ERR; bytes_var_to_copy = *end_offset - *start_offset; } else { bytes_var_to_copy = tiles_var_sizes_[attribute_id] - *start_offset; } // If bytes do not fit in variable buffer, we need to adjust if(bytes_var_to_copy > buffer_var_free_space) { // Perform binary search int64_t min = start_cell_pos + 1; int64_t max = end_cell_pos; int64_t med; // Invariants: // (tile[min-1] - tile[start_cell_pos]) < buffer_var_free_space AND // (tile[max+1] - tile[start_cell_pos]) > buffer_var_free_space while(min <= max) { med = min + ((max - min) / 2); // Calculate variable bytes to copy if(GET_CELL_PTR_FROM_OFFSET_TILE( attribute_id, med, med_offset) != TILEDB_RS_OK) return TILEDB_RS_ERR; bytes_var_to_copy = *med_offset - *start_offset; // Check condition if(bytes_var_to_copy > buffer_var_free_space) max = med-1; else if(bytes_var_to_copy < buffer_var_free_space) min = med+1; else break; } // Determine the end position of the range int64_t tmp_end = -1; if(min > max) tmp_end = min - 2 ; else tmp_end = med - 1; end_cell_pos = std::max(tmp_end, start_cell_pos-1); // Update variable bytes to copy if(GET_CELL_PTR_FROM_OFFSET_TILE( attribute_id, end_cell_pos+1, end_offset) != TILEDB_RS_OK) return TILEDB_RS_ERR; bytes_var_to_copy = *end_offset - *start_offset; } // Update bytes to copy bytes_to_copy = (end_cell_pos - start_cell_pos + 1) * TILEDB_CELL_VAR_OFFSET_SIZE; // Sanity checks assert(bytes_to_copy <= buffer_free_space); assert(bytes_var_to_copy <= buffer_var_free_space); return TILEDB_RS_OK; } void ReadState::compute_tile_search_range() { // For easy reference int coords_type = array_schema_->coords_type(); // Applicable only to sparse fragments if(fragment_->dense()) return; // Invoke the proper templated function if(coords_type == TILEDB_INT32) { compute_tile_search_range(); } else if(coords_type == TILEDB_INT64) { compute_tile_search_range(); } else if(coords_type == TILEDB_FLOAT32) { compute_tile_search_range(); } else if(coords_type == TILEDB_FLOAT64) { compute_tile_search_range(); } else { // The code should never reach here assert(0); } } template void ReadState::compute_tile_search_range() { // For easy reference int cell_order = array_schema_->cell_order(); // Initialize the tile search range if(cell_order == TILEDB_HILBERT) // HILBERT CELL ORDER compute_tile_search_range_hil(); else // COLUMN- OR ROW-MAJOR compute_tile_search_range_col_or_row(); // Handle no overlap if(tile_search_range_[0] == -1 || tile_search_range_[1] == -1) done_ = true; } template void ReadState::compute_tile_search_range_col_or_row() { // For easy reference int dim_num = array_schema_->dim_num(); const T* subarray = static_cast(array_->subarray()); int64_t tile_num = book_keeping_->tile_num(); const std::vector& bounding_coords = book_keeping_->bounding_coords(); // Calculate subarray coordinates T* subarray_min_coords = new T[dim_num]; T* subarray_max_coords = new T[dim_num]; for(int i=0; i(bounding_coords[med]); tile_end_coords = &(static_cast(bounding_coords[med])[dim_num]); // Calculate precedence if(array_schema_->tile_cell_order_cmp( subarray_min_coords, tile_start_coords) < 0) { // Subarray min precedes MBR max = med-1; } else if(array_schema_->tile_cell_order_cmp( subarray_min_coords, tile_end_coords) > 0) { // Subarray min succeeds MBR min = med+1; } else { // Subarray min in MBR break; } } bool is_unary = is_unary_subarray(subarray, dim_num); // Determine the start position of the range if(max < min) // Subarray min precedes the tile at position min tile_search_range_[0] = (is_unary) ? -1 : min; else // Subarray min included in a tile tile_search_range_[0] = med; if(is_unary) { // Unary range // The end position is the same as the start tile_search_range_[1] = tile_search_range_[0]; } else { // Need to find the end position // --- Finding the end tile in search range // Perform binary search min = 0; max = tile_num - 1; while(min <= max) { med = min + ((max - min) / 2); // Get info for bounding coordinates tile_start_coords = static_cast(bounding_coords[med]); tile_end_coords = &(static_cast(bounding_coords[med])[dim_num]); // Calculate precedence if(array_schema_->tile_cell_order_cmp( subarray_max_coords, tile_start_coords) < 0) { // Subarray max precedes MBR max = med-1; } else if(array_schema_->tile_cell_order_cmp( subarray_max_coords, tile_end_coords) > 0) { // Subarray max succeeds MBR min = med+1; } else { // Subarray max in MBR break; } } // Determine the start position of the range if(max < min) // Subarray max succeeds the tile at position max tile_search_range_[1] = max; else // Subarray max included in a tile tile_search_range_[1] = med; } // No overlap if(tile_search_range_[1] < tile_search_range_[0]) { tile_search_range_[0] = -1; tile_search_range_[1] = -1; } // Clean up delete [] subarray_min_coords; delete [] subarray_max_coords; } template void ReadState::compute_tile_search_range_hil() { // For easy reference int dim_num = array_schema_->dim_num(); const T* subarray = static_cast(array_->subarray()); int64_t tile_num = book_keeping_->tile_num(); if(is_unary_subarray(subarray, dim_num)) { // Unary range // For easy reference const std::vector& bounding_coords = book_keeping_->bounding_coords(); // Calculate range coordinates T* subarray_coords = new T[dim_num]; for(int i=0; i(bounding_coords[med]); tile_end_coords = &(static_cast(bounding_coords[med])[dim_num]); // Calculate precedence if(array_schema_->tile_cell_order_cmp( subarray_coords, tile_start_coords) < 0) { // Unary subarray precedes MBR max = med-1; } else if(array_schema_->tile_cell_order_cmp( subarray_coords, tile_end_coords) > 0) { // Unary subarray succeeds MBR min = med+1; } else { // Unary subarray inside MBR break; } } // Determine the start position of the range if(max < min) // Unary subarray not inside a tile tile_search_range_[0] = -1; else // Unary subarray inside a tile tile_search_range_[0] = med; // For unary ranges, start and end positions are the same tile_search_range_[1] = tile_search_range_[0]; // Clean up delete [] subarray_coords; } else { // Non-unary range if(book_keeping_->tile_num() > 0) { tile_search_range_[0] = 0; tile_search_range_[1] = book_keeping_->tile_num() - 1; } else { tile_search_range_[0] = -1; tile_search_range_[1] = -1; } } } int ReadState::decompress_tile( int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size, bool decompress_offsets) { Codec* codec; if (decompress_offsets) { codec = offsets_codec_[attribute_id]; if (codec == NULL) { tile = tile_compressed; tile_size = tile_compressed_size; return TILEDB_RS_OK; } } else { codec = codec_[attribute_id]; } if(codec->decompress_tile(tile_compressed, tile_compressed_size, tile, tile_size) != TILEDB_CD_OK) { std::string errmsg = "Cannot decompress tile for " + construct_filename(attribute_id, decompress_offsets); PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } return TILEDB_RS_OK; } template int64_t ReadState::get_cell_pos_after(const T* coords) { // For easy reference int64_t cell_num = book_keeping_->cell_num(fetched_tile_[attribute_num_+1]); // Perform binary search to find the position of coords in the tile int64_t min = 0; int64_t max = cell_num - 1; int64_t med; int cmp; const void* coords_t; while(min <= max) { med = min + ((max - min) / 2); // Update search range if(GET_COORDS_PTR_FROM_SEARCH_TILE(med, coords_t) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Compute order cmp = array_schema_->tile_cell_order_cmp( coords, static_cast(coords_t)); if(cmp < 0) max = med-1; else if(cmp > 0) min = med+1; else break; } // Return if(max < min) return min; // After else return med + 1; // After (med is at) } template int64_t ReadState::get_cell_pos_at_or_after(const T* coords) { // For easy reference int64_t cell_num = book_keeping_->cell_num(fetched_tile_[attribute_num_+1]); // Perform binary search to find the position of coords in the tile int64_t min = 0; int64_t max = cell_num - 1; int64_t med; int cmp; const void* coords_t; while(min <= max) { med = min + ((max - min) / 2); // Update search range if(GET_COORDS_PTR_FROM_SEARCH_TILE(med, coords_t) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Compute order cmp = array_schema_->tile_cell_order_cmp( coords, static_cast(coords_t)); if(cmp < 0) max = med-1; else if(cmp > 0) min = med+1; else break; } // Return if(max < min) return min; // After else return med; // At } template int64_t ReadState::get_cell_pos_at_or_before(const T* coords) { // For easy reference int64_t cell_num = book_keeping_->cell_num(fetched_tile_[attribute_num_+1]); // Perform binary search to find the position of coords in the tile int64_t min = 0; int64_t max = cell_num - 1; int64_t med; int cmp; const void* coords_t; while(min <= max) { med = min + ((max - min) / 2); // Update search range if(GET_COORDS_PTR_FROM_SEARCH_TILE(med, coords_t) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Compute order cmp = array_schema_->tile_cell_order_cmp( coords, static_cast(coords_t)); if(cmp < 0) max = med-1; else if(cmp > 0) min = med+1; else break; } // Return if(max < min) return max; // Before else return med; // At } inline int ReadState::GET_COORDS_PTR_FROM_SEARCH_TILE( int64_t i, const void*& coords) { // For easy reference char* tile = static_cast(tiles_[attribute_num_+1]); // The tile is in main memory if(tile != NULL) { coords = tile + i*coords_size_; return TILEDB_RS_OK; } // Read from file if (read_segment(attribute_num_, false, tiles_file_offsets_[attribute_num_+1] + i*coords_size_, tmp_coords_, coords_size_) == TILEDB_RS_ERR) { return TILEDB_RS_ERR; } // Get coordinates pointer coords = tmp_coords_; // Success return TILEDB_RS_OK; } int ReadState::GET_CELL_PTR_FROM_OFFSET_TILE( int attribute_id, int64_t i, const size_t*& offset) { // For easy reference char* tile = static_cast(tiles_[attribute_id]); // The tile is in main memory if(tile != NULL) { offset = (const size_t*) (tile + i*sizeof(size_t)); return TILEDB_RS_OK; } // Read attribute if (read_segment(attribute_id, false, tiles_file_offsets_[attribute_id] + i*sizeof(size_t), &tmp_offset_, sizeof(size_t)) == TILEDB_RS_ERR) { return TILEDB_RS_ERR; } // Get coordinates pointer offset = tiles_file_offsets_[attribute_id] + &tmp_offset_; // Success return TILEDB_RS_OK; } bool ReadState::is_empty_attribute(int attribute_id) const { // Special case for search coordinate tiles if(attribute_id == attribute_num_ + 1) attribute_id = attribute_num_; return file_size_[attribute_id] == TILEDB_FS_ERR; } int ReadState::map_tile_from_file_cmp( int attribute_id, off_t offset, size_t tile_size) { // To handle the special case of the search tile // The real attribute id corresponds to an actual attribute or coordinates int attribute_id_real = (attribute_id == attribute_num_+1) ? attribute_num_ : attribute_id; // Unmap if(map_addr_compressed_ != NULL) { if(munmap(map_addr_compressed_, map_addr_compressed_length_)) { std::string errmsg = "Cannot read tile from file with map; Memory unmap error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } } // Prepare attribute file name std::string filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id_real) + TILEDB_FILE_SUFFIX; // Calculate offset considering the page size size_t page_size = sysconf(_SC_PAGE_SIZE); off_t start_offset = (offset / page_size) * page_size; size_t extra_offset = offset - start_offset; size_t new_length = tile_size + extra_offset; // Open file int fd = open(filename.c_str(), O_RDONLY); if(fd == -1) { munmap(map_addr_compressed_, map_addr_compressed_length_); map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; tile_compressed_ = NULL; std::string errmsg = "Cannot read tile from file; File opening error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } // Map map_addr_compressed_ = mmap( map_addr_compressed_, new_length, PROT_READ, MAP_SHARED, fd, start_offset); if(map_addr_compressed_ == MAP_FAILED) { map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; tile_compressed_ = NULL; std::string errmsg = "Cannot read tile from file; Memory map error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } map_addr_compressed_length_ = new_length; // Set properly the compressed tile pointer tile_compressed_ = static_cast(map_addr_compressed_) + extra_offset; // Close file if(close(fd)) { munmap(map_addr_compressed_, map_addr_compressed_length_); map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; tile_compressed_ = NULL; std::string errmsg = "Cannot read tile from file; File closing error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } return TILEDB_RS_OK; } int ReadState::map_tile_from_file_var_cmp( int attribute_id, off_t offset, size_t tile_size) { // Unmap if(map_addr_compressed_ != NULL) { if(munmap(map_addr_compressed_, map_addr_compressed_length_)) { std::string errmsg = "Cannot read tile from file with map; Memory unmap error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } } // Prepare attribute file name std::string filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id) + "_var" + TILEDB_FILE_SUFFIX; // Calculate offset considering the page size size_t page_size = sysconf(_SC_PAGE_SIZE); off_t start_offset = (offset / page_size) * page_size; size_t extra_offset = offset - start_offset; size_t new_length = tile_size + extra_offset; // Open file int fd = open(filename.c_str(), O_RDONLY); if(fd == -1) { munmap(map_addr_compressed_, map_addr_compressed_length_); map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; tile_compressed_ = NULL; std::string errmsg = "Cannot read tile from file; File opening error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } // Map // new_length could be 0 for variable length fields, mmap will fail // if new_length == 0 if(new_length > 0u) { map_addr_compressed_ = mmap( map_addr_compressed_, new_length, PROT_READ, MAP_SHARED, fd, start_offset); if(map_addr_compressed_ == MAP_FAILED) { map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; tile_compressed_ = NULL; std::string errmsg = "Cannot read tile from file; Memory map error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } } else { map_addr_var_[attribute_id] = 0; } map_addr_compressed_length_ = new_length; // Set properly the compressed tile pointer tile_compressed_ = static_cast(map_addr_compressed_) + extra_offset; // Close file if(close(fd)) { munmap(map_addr_compressed_, map_addr_compressed_length_); map_addr_compressed_ = NULL; map_addr_compressed_length_ = 0; tile_compressed_ = NULL; std::string errmsg = "Cannot read tile from file; File closing error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } return TILEDB_RS_OK; } int ReadState::map_tile_from_file_cmp_none( int attribute_id, off_t offset, size_t tile_size) { // To handle the special case of the search tile // The real attribute id corresponds to an actual attribute or coordinates int attribute_id_real = (attribute_id == attribute_num_+1) ? attribute_num_ : attribute_id; // Unmap if(map_addr_[attribute_id] != NULL) { if(munmap(map_addr_[attribute_id], map_addr_lengths_[attribute_id])) { std::string errmsg = "Cannot read tile from file with map; Memory unmap error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } } // Prepare attribute file name std::string filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id_real) + TILEDB_FILE_SUFFIX; // Calculate offset considering the page size size_t page_size = sysconf(_SC_PAGE_SIZE); off_t start_offset = (offset / page_size) * page_size; size_t extra_offset = offset - start_offset; size_t new_length = tile_size + extra_offset; // Open file int fd = open(filename.c_str(), O_RDONLY); if(fd == -1) { map_addr_[attribute_id] = NULL; map_addr_lengths_[attribute_id] = 0; tiles_[attribute_id] = NULL; tiles_sizes_[attribute_id] = 0; std::string errmsg = "Cannot read tile from file; File opening error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } // Map bool var_size = array_schema_->var_size(attribute_id_real); int prot = var_size ? (PROT_READ | PROT_WRITE) : PROT_READ; int flags = var_size ? MAP_PRIVATE : MAP_SHARED; map_addr_[attribute_id] = mmap(map_addr_[attribute_id], new_length, prot, flags, fd, start_offset); if(map_addr_[attribute_id] == MAP_FAILED) { map_addr_[attribute_id] = NULL; map_addr_lengths_[attribute_id] = 0; tiles_[attribute_id] = NULL; tiles_sizes_[attribute_id] = 0; std::string errmsg = "Cannot read tile from file; Memory map error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } map_addr_lengths_[attribute_id] = new_length; // Set properly the tile pointer tiles_[attribute_id] = static_cast(map_addr_[attribute_id]) + extra_offset; // Close file if(close(fd)) { munmap(map_addr_[attribute_id], map_addr_lengths_[attribute_id]); map_addr_[attribute_id] = NULL; map_addr_lengths_[attribute_id] = 0; tiles_[attribute_id] = NULL; tiles_sizes_[attribute_id] = 0; std::string errmsg = "Cannot read tile from file; File closing error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } return TILEDB_RS_OK; } int ReadState::map_tile_from_file_var_cmp_none( int attribute_id, off_t offset, size_t tile_size) { // Unmap if(map_addr_var_[attribute_id] != NULL) { if(munmap( map_addr_var_[attribute_id], map_addr_var_lengths_[attribute_id])) { std::string errmsg = "Cannot read tile from file with map; Memory unmap error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } } // Prepare attribute file name std::string filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id) + "_var" + TILEDB_FILE_SUFFIX; // Calculate offset considering the page size size_t page_size = sysconf(_SC_PAGE_SIZE); off_t start_offset = (offset / page_size) * page_size; size_t extra_offset = offset - start_offset; size_t new_length = tile_size + extra_offset; // Open file int fd = open(filename.c_str(), O_RDONLY); if(fd == -1) { map_addr_var_[attribute_id] = NULL; map_addr_var_lengths_[attribute_id] = 0; tiles_var_[attribute_id] = NULL; tiles_var_sizes_[attribute_id] = 0; std::string errmsg = "Cannot read tile from file; File opening error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } // Map // new_length could be 0 for variable length fields, mmap will fail // if new_length == 0 if(new_length > 0u) { map_addr_var_[attribute_id] = mmap( map_addr_var_[attribute_id], new_length, PROT_READ, MAP_SHARED, fd, start_offset); if(map_addr_var_[attribute_id] == MAP_FAILED) { map_addr_var_[attribute_id] = NULL; map_addr_var_lengths_[attribute_id] = 0; tiles_var_[attribute_id] = NULL; tiles_var_sizes_[attribute_id] = 0; std::string errmsg = "Cannot read tile from file; Memory map error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } } else { map_addr_var_[attribute_id] = 0; } map_addr_var_lengths_[attribute_id] = new_length; // Set properly the tile pointer tiles_var_[attribute_id] = static_cast(map_addr_var_[attribute_id]) + extra_offset; tiles_var_sizes_[attribute_id] = tile_size; // Close file if(close(fd)) { munmap(map_addr_var_[attribute_id], map_addr_var_lengths_[attribute_id]); map_addr_var_[attribute_id] = NULL; map_addr_var_lengths_[attribute_id] = 0; tiles_var_[attribute_id] = NULL; tiles_var_sizes_[attribute_id] = 0; std::string errmsg = "Cannot read tile from file; File closing error"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; } // Success return TILEDB_RS_OK; } #ifdef HAVE_MPI int ReadState::mpi_io_read_tile_from_file_cmp( int attribute_id, off_t offset, size_t tile_size) { // For easy reference const MPI_Comm* mpi_comm = array_->config()->mpi_comm(); // To handle the special case of the search tile // The real attribute id corresponds to an actual attribute or coordinates int attribute_id_real = (attribute_id == attribute_num_+1) ? attribute_num_ : attribute_id; // Potentially allocate compressed tile buffer if(tile_compressed_ == NULL) { size_t full_tile_size = fragment_->tile_size(attribute_id_real); size_t tile_max_size = full_tile_size + 6 + 5*(ceil(full_tile_size/16834.0)); tile_compressed_ = malloc(tile_max_size); tile_compressed_allocated_size_ = tile_max_size; } // Prepare attribute file name std::string filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id_real) + TILEDB_FILE_SUFFIX; // Read from file if(mpi_io_read_from_file( mpi_comm, filename, offset, tile_compressed_, tile_size) != TILEDB_UT_OK) { tiledb_rs_errmsg = tiledb_ut_errmsg; return TILEDB_RS_ERR; } // Success return TILEDB_RS_OK; } int ReadState::mpi_io_read_tile_from_file_var_cmp( int attribute_id, off_t offset, size_t tile_size) { // For easy reference const MPI_Comm* mpi_comm = array_->config()->mpi_comm(); // Potentially allocate compressed tile buffer if(tile_compressed_ == NULL) { tile_compressed_ = malloc(tile_size); tile_compressed_allocated_size_ = tile_size; } // Potentially expand compressed tile buffer if(tile_compressed_allocated_size_ < tile_size) { tile_compressed_ = realloc(tile_compressed_, tile_size); tile_compressed_allocated_size_ = tile_size; } // Prepare attribute file name std::string filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id) + "_var" + TILEDB_FILE_SUFFIX; // Read from file if(mpi_io_read_from_file( mpi_comm, filename, offset, tile_compressed_, tile_size) != TILEDB_UT_OK) { tiledb_rs_errmsg = tiledb_ut_errmsg; return TILEDB_RS_ERR; } // Success return TILEDB_RS_OK; } #endif int ReadState::prepare_tile_for_reading( int attribute_id, int64_t tile_i) { // For easy reference int compression = array_schema_->compression(attribute_id); // Invoke the proper function based on the compression type if(compression == TILEDB_NO_COMPRESSION) return prepare_tile_for_reading_cmp_none(attribute_id, tile_i); else // All compressions return prepare_tile_for_reading_cmp(attribute_id, tile_i); } int ReadState::prepare_tile_for_reading_var( int attribute_id, int64_t tile_i) { // For easy reference int compression = array_schema_->compression(attribute_id); // Invoke the proper function based on the compression type if(compression == TILEDB_NO_COMPRESSION) return prepare_tile_for_reading_var_cmp_none(attribute_id, tile_i); else // All compressions return prepare_tile_for_reading_var_cmp(attribute_id, tile_i); } int ReadState::prepare_tile_for_reading_cmp( int attribute_id, int64_t tile_i) { // Return if the tile has already been fetched if(tile_i == fetched_tile_[attribute_id]) return TILEDB_RS_OK; // To handle the special case of the search tile // The real attribute id corresponds to an actual attribute or coordinates int attribute_id_real = (attribute_id == attribute_num_+1) ? attribute_num_ : attribute_id; // For easy reference size_t cell_size = array_schema_->cell_size(attribute_id_real); size_t full_tile_size = fragment_->tile_size(attribute_id_real); int64_t cell_num = book_keeping_->cell_num(tile_i); size_t tile_size = cell_num * cell_size; const std::vector >& tile_offsets = book_keeping_->tile_offsets(); int64_t tile_num = book_keeping_->tile_num(); // Allocate space for the tile if needed if(tiles_[attribute_id] == NULL) tiles_[attribute_id] = malloc(full_tile_size); // Find file offset where the tile begins off_t file_offset = tile_offsets[attribute_id_real][tile_i]; auto file_size = file_size_[attribute_id_real]; assert(file_size != TILEDB_FS_ERR); size_t tile_compressed_size = (tile_i == tile_num-1) ? file_size - tile_offsets[attribute_id_real][tile_i] : tile_offsets[attribute_id_real][tile_i+1] - tile_offsets[attribute_id_real][tile_i]; // Read tile from file int rc = TILEDB_RS_OK; int read_method = array_->config()->read_method(); if(read_method == TILEDB_IO_READ) { rc = read_tile_from_file_cmp( attribute_id, file_offset, tile_compressed_size); } else if(read_method == TILEDB_IO_MMAP) { rc = map_tile_from_file_cmp( attribute_id, file_offset, tile_compressed_size); } else if(read_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_read_tile_from_file_cmp( attribute_id, file_offset, tile_compressed_size); #else // Error: MPI not supported std::string errmsg = "Cannot prepare tile for reading (gzip); MPI not supported"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; #endif } // Error if(rc != TILEDB_RS_OK) return TILEDB_RS_ERR; // Decompress tile if(decompress_tile( attribute_id, static_cast(tile_compressed_), tile_compressed_size, static_cast(tiles_[attribute_id]), full_tile_size) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Set the tile size tiles_sizes_[attribute_id] = tile_size; // Set tile offset tiles_offsets_[attribute_id] = 0; // Mark as fetched fetched_tile_[attribute_id] = tile_i; // Success return TILEDB_RS_OK; } int ReadState::prepare_tile_for_reading_cmp_none( int attribute_id, int64_t tile_i) { // Return if the tile has already been fetched if(tile_i == fetched_tile_[attribute_id]) return TILEDB_RS_OK; // To handle the special case of the search tile // The real attribute id corresponds to an actual attribute or coordinates int attribute_id_real = (attribute_id == attribute_num_+1) ? attribute_num_ : attribute_id; // For easy reference size_t cell_size = array_schema_->cell_size(attribute_id_real); size_t full_tile_size = fragment_->tile_size(attribute_id_real); int64_t cell_num = book_keeping_->cell_num(tile_i); size_t tile_size = cell_num * cell_size; // Find file offset where the tile begins off_t file_offset = tile_i * full_tile_size; // Read tile from file int rc = TILEDB_RS_OK; int read_method = array_->config()->read_method(); if(read_method == TILEDB_IO_READ || read_method == TILEDB_IO_MPI) rc = set_tile_file_offset( attribute_id, file_offset); else if(read_method == TILEDB_IO_MMAP) rc = map_tile_from_file_cmp_none( attribute_id, file_offset, tile_size); // Error if(rc != TILEDB_RS_OK) return TILEDB_RS_ERR; // Set the tile size tiles_sizes_[attribute_id] = tile_size; // Set tile offset tiles_offsets_[attribute_id] = 0; // Mark as fetched fetched_tile_[attribute_id] = tile_i; // Success return TILEDB_RS_OK; } int ReadState::prepare_tile_for_reading_var_cmp( int attribute_id, int64_t tile_i) { // Return if the tile has already been fetched if(tile_i == fetched_tile_[attribute_id]) return TILEDB_RS_OK; // Sanity check assert( attribute_id < attribute_num_ && array_schema_->var_size(attribute_id)); // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; size_t full_tile_size = fragment_->tile_size(attribute_id); int64_t cell_num = book_keeping_->cell_num(tile_i); size_t tile_size = cell_num * cell_size; const std::vector >& tile_offsets = book_keeping_->tile_offsets(); const std::vector >& tile_var_offsets = book_keeping_->tile_var_offsets(); int64_t tile_num = book_keeping_->tile_num(); // ========== Get tile with variable cell offsets ========== // // Find file offset where the tile begins off_t file_offset = tile_offsets[attribute_id][tile_i]; auto file_size = file_size_[attribute_id]; assert(file_size != TILEDB_FS_ERR); size_t tile_compressed_size = (tile_i == tile_num-1) ? file_size - tile_offsets[attribute_id][tile_i] : tile_offsets[attribute_id][tile_i+1] - tile_offsets[attribute_id][tile_i]; // Allocate space for the tile if needed if(tiles_[attribute_id] == NULL) tiles_[attribute_id] = malloc(full_tile_size); // Read tile from file int rc = TILEDB_RS_OK; int read_method = array_->config()->read_method(); if(read_method == TILEDB_IO_READ) { rc = read_tile_from_file_cmp( attribute_id, file_offset, tile_compressed_size); } else if(read_method == TILEDB_IO_MMAP) { rc = map_tile_from_file_cmp( attribute_id, file_offset, tile_compressed_size); } else if(read_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_read_tile_from_file_cmp( attribute_id, file_offset, tile_compressed_size); #else // Error: MPI not supported std::string errmsg = "Cannot prepare variable tile for reading (gzip); MPI not supported"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; #endif } // Error if(rc != TILEDB_RS_OK) return TILEDB_RS_ERR; // Decompress tile if(decompress_tile( attribute_id, static_cast(tile_compressed_), tile_compressed_size, static_cast(tiles_[attribute_id]), tile_size, true) != TILEDB_RS_OK) return TILEDB_RS_ERR; // Set the tile size tiles_sizes_[attribute_id] = tile_size; // Update tile offset tiles_offsets_[attribute_id] = 0; // ========== Get variable tile ========== // // Calculate offset and compressed tile size file_offset = tile_var_offsets[attribute_id][tile_i]; file_size = file_var_size_[attribute_id]; assert(file_size != TILEDB_FS_ERR); tile_compressed_size = (tile_i == tile_num-1) ? file_size-tile_var_offsets[attribute_id][tile_i] : tile_var_offsets[attribute_id][tile_i+1] - tile_var_offsets[attribute_id][tile_i]; // Get size of decompressed tile size_t tile_var_size = book_keeping_->tile_var_sizes()[attribute_id][tile_i]; //Non-empty tile, decompress if(tile_var_size > 0u) { // Potentially allocate space for buffer if(tiles_var_[attribute_id] == NULL) { tiles_var_[attribute_id] = malloc(tile_var_size); tiles_var_allocated_size_[attribute_id] = tile_var_size; } // Potentially expand buffer if(tile_var_size > tiles_var_allocated_size_[attribute_id]) { tiles_var_[attribute_id] = realloc(tiles_var_[attribute_id], tile_var_size); tiles_var_allocated_size_[attribute_id] = tile_var_size; } // Read tile from file int rc = TILEDB_RS_OK; int read_method = array_->config()->read_method(); if(read_method == TILEDB_IO_READ) { rc = read_tile_from_file_var_cmp( attribute_id, file_offset, tile_compressed_size); } else if(read_method == TILEDB_IO_MMAP) { rc = map_tile_from_file_var_cmp( attribute_id, file_offset, tile_compressed_size); } else if(read_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_read_tile_from_file_var_cmp( attribute_id, file_offset, tile_compressed_size); #else // Error: MPI not supported std::string errmsg = "Cannot prepare variable tile for reading (gzip); MPI not supported"; PRINT_ERROR(errmsg); tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; return TILEDB_RS_ERR; #endif } // Error if(rc != TILEDB_RS_OK) return TILEDB_RS_ERR; // Decompress tile if(decompress_tile( attribute_id, static_cast(tile_compressed_), tile_compressed_size, static_cast(tiles_var_[attribute_id]), tile_var_size) != TILEDB_RS_OK) return TILEDB_RS_ERR; } // Set the variable tile size tiles_var_sizes_[attribute_id] = tile_var_size; // Set the variable tile offset tiles_var_offsets_[attribute_id] = 0; // Shift variable cell offsets shift_var_offsets(attribute_id); // Mark as fetched fetched_tile_[attribute_id] = tile_i; // Success return TILEDB_RS_OK; } int ReadState::prepare_tile_for_reading_var_cmp_none( int attribute_id, int64_t tile_i) { // Return if the tile has already been fetched if(tile_i == fetched_tile_[attribute_id]) return TILEDB_RS_OK; // Sanity check assert( attribute_id < attribute_num_ && array_schema_->var_size(attribute_id)); // For easy reference size_t full_tile_size = fragment_->tile_size(attribute_id); int64_t cell_num = book_keeping_->cell_num(tile_i); size_t tile_size = cell_num * TILEDB_CELL_VAR_OFFSET_SIZE; int64_t tile_num = book_keeping_->tile_num(); off_t file_offset = tile_i * full_tile_size; // Read tile from file int rc = TILEDB_RS_OK; int read_method = array_->config()->read_method(); if(read_method == TILEDB_IO_READ || read_method == TILEDB_IO_MPI) rc = set_tile_file_offset( attribute_id, file_offset); else if(read_method == TILEDB_IO_MMAP) rc = map_tile_from_file_cmp_none( attribute_id, file_offset, tile_size); // Error if(rc != TILEDB_RS_OK) return TILEDB_RS_ERR; // Set tile size tiles_sizes_[attribute_id] = tile_size; // Calculate the start and end offsets for the variable-sized tile, // as well as the variable tile size const size_t* tile_s; if(GET_CELL_PTR_FROM_OFFSET_TILE( attribute_id, 0, tile_s) != TILEDB_RS_OK) return TILEDB_RS_ERR; off_t start_tile_var_offset = *tile_s; off_t end_tile_var_offset = 0; size_t tile_var_size; if(tile_i != tile_num - 1) { // Not the last tile if (read_segment(attribute_id, false, file_offset + full_tile_size, &end_tile_var_offset, TILEDB_CELL_VAR_OFFSET_SIZE) == TILEDB_RS_ERR) { return TILEDB_RS_ERR; } tile_var_size = end_tile_var_offset - tile_s[0]; } else { // Last tile tile_var_size = file_var_size_[attribute_id] - tile_s[0]; } // Read tile from file if(read_method == TILEDB_IO_READ || read_method == TILEDB_IO_MPI) rc = set_tile_var_file_offset( attribute_id, start_tile_var_offset); else if(read_method == TILEDB_IO_MMAP) rc = map_tile_from_file_var_cmp_none( attribute_id, start_tile_var_offset, tile_var_size); if(rc != TILEDB_RS_OK) return TILEDB_RS_ERR; // Set offsets tiles_offsets_[attribute_id] = 0; tiles_var_offsets_[attribute_id] = 0; // Set variable tile size tiles_var_sizes_[attribute_id] = tile_var_size; // Shift starting offsets of variable-sized cells shift_var_offsets(attribute_id); // Mark as fetched fetched_tile_[attribute_id] = tile_i; // Success return TILEDB_RS_OK; } int ReadState::READ_FROM_TILE( int attribute_id, void* buffer, size_t tile_offset, size_t bytes_to_copy) { // For easy reference char* tile = static_cast(tiles_[attribute_id]); // The tile is in main memory if(tile != NULL) { memcpy(buffer, tile + tile_offset, bytes_to_copy); return TILEDB_RS_OK; } // We need to read from the disk return read_segment(attribute_id, false, tiles_file_offsets_[attribute_id] + tile_offset, buffer, bytes_to_copy); } int ReadState::READ_FROM_TILE_VAR( int attribute_id, void* buffer, size_t tile_offset, size_t bytes_to_copy) { // For easy reference char* tile = static_cast(tiles_var_[attribute_id]); // The tile is in main memory if(tile != NULL) { memcpy(buffer, tile + tile_offset, bytes_to_copy); return TILEDB_RS_OK; } // We need to read from the disk return read_segment(attribute_id, true, tile_offset, buffer, bytes_to_copy); } int ReadState::read_tile_from_file_cmp( int attribute_id, off_t offset, size_t tile_size) { // To handle the special case of the search tile // The real attribute id corresponds to an actual attribute or coordinates int attribute_id_real = (attribute_id == attribute_num_+1) ? attribute_num_ : attribute_id; // Potentially allocate compressed tile buffer if(tile_compressed_ == NULL) { tile_compressed_ = malloc(tile_size); tile_compressed_allocated_size_ = tile_size; } // Potentially expand compressed tile buffer if(tile_compressed_allocated_size_ < tile_size) { tile_compressed_ = realloc(tile_compressed_, tile_size); tile_compressed_allocated_size_ = tile_size; } // Read from file return read_segment(attribute_id_real, false, offset, tile_compressed_, tile_size); } int ReadState::read_tile_from_file_var_cmp( int attribute_id, off_t offset, size_t tile_size) { // Potentially allocate compressed tile buffer if(tile_compressed_ == NULL) { tile_compressed_ = malloc(tile_size); tile_compressed_allocated_size_ = tile_size; } // Potentially expand compressed tile buffer if(tile_compressed_allocated_size_ < tile_size) { tile_compressed_ = realloc(tile_compressed_, tile_size); tile_compressed_allocated_size_ = tile_size; } return read_segment(attribute_id, true, offset, tile_compressed_, tile_size); } int ReadState::set_tile_file_offset( int attribute_id, off_t offset) { // Set file offset tiles_file_offsets_[attribute_id] = offset; // Success return TILEDB_RS_OK; } int ReadState::set_tile_var_file_offset( int attribute_id, off_t offset) { // Set file offset tiles_var_file_offsets_[attribute_id] = offset; // Success return TILEDB_RS_OK; } void ReadState::shift_var_offsets(int attribute_id) { // For easy reference int64_t cell_num = tiles_sizes_[attribute_id] / TILEDB_CELL_VAR_OFFSET_SIZE; size_t* tile_s = static_cast(tiles_[attribute_id]); if (tile_s) { size_t first_offset = tile_s[0]; // Shift offsets for(int64_t i=0; i(buffer); size_t start_offset = buffer_s[0]; // Shift offsets for(int64_t i=0; i( const int* coords, int* coords_after, bool& coords_retrieved); template int ReadState::get_coords_after( const int64_t* coords, int64_t* coords_after, bool& coords_retrieved); template int ReadState::get_coords_after( const float* coords, float* coords_after, bool& coords_retrieved); template int ReadState::get_coords_after( const double* coords, double* coords_after, bool& coords_retrieved); template int ReadState::get_enclosing_coords( int tile_i, const int* target_coords, const int* start_coords, const int* end_coords, int* left_coords, int* right_coords, bool& left_retrieved, bool& right_retrieved, bool& target_exists); template int ReadState::get_enclosing_coords( int tile_i, const int64_t* target_coords, const int64_t* start_coords, const int64_t* end_coords, int64_t* left_coords, int64_t* right_coords, bool& left_retrieved, bool& right_retrieved, bool& target_exists); template int ReadState::get_enclosing_coords( int tile_i, const float* target_coords, const float* start_coords, const float* end_coords, float* left_coords, float* right_coords, bool& left_retrieved, bool& right_retrieved, bool& target_exists); template int ReadState::get_enclosing_coords( int tile_i, const double* target_coords, const double* start_coords, const double* end_coords, double* left_coords, double* right_coords, bool& left_retrieved, bool& right_retrieved, bool& target_exists); template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const int* cell_range, FragmentCellPosRange& fragment_cell_pos_range); template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const int64_t* cell_range, FragmentCellPosRange& fragment_cell_pos_range); template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const float* cell_range, FragmentCellPosRange& fragment_cell_pos_range); template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const double* cell_range, FragmentCellPosRange& fragment_cell_pos_range); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, const int* start_coords, const int* end_coords, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, const int64_t* start_coords, const int64_t* end_coords, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, const float* start_coords, const float* end_coords, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, const double* start_coords, const double* end_coords, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_dense( int fragment_i, FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_dense( int fragment_i, FragmentCellRanges& fragment_cell_ranges); template void ReadState::get_next_overlapping_tile_dense( const int* tile_coords); template void ReadState::get_next_overlapping_tile_dense( const int64_t* tile_coords); template void ReadState::get_next_overlapping_tile_sparse( const int* tile_coords); template void ReadState::get_next_overlapping_tile_sparse( const int64_t* tile_coords); template void ReadState::get_next_overlapping_tile_sparse(); template void ReadState::get_next_overlapping_tile_sparse(); template void ReadState::get_next_overlapping_tile_sparse(); template void ReadState::get_next_overlapping_tile_sparse(); genomicsdb-0.0~git20231212.9d7ddd0/core/src/fragment/write_state.cc000066400000000000000000002007441453617025200245370ustar00rootroot00000000000000/** * @file write_state.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the WriteState class. */ #include "comparators.h" #include "storage_buffer.h" #include "tiledb_constants.h" #include "utils.h" #include "write_state.h" #include #include #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_WS_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif #if defined HAVE_OPENMP && defined USE_PARALLEL_SORT #include #define SORT(first, last, comp) __gnu_parallel::sort((first), (last), (comp)) #else #include #define SORT(first, last, comp) std::sort((first), (last), (comp)) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_ws_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ WriteState::WriteState( const Fragment* fragment, BookKeeping* book_keeping) : book_keeping_(book_keeping), fragment_(fragment) { // For easy reference array_ = fragment_->array(); array_schema_ = array_->array_schema(); attribute_num_ = array_schema_->attribute_num(); size_t coords_size = array_schema_->coords_size(); // Initialize the number of cells written in the current tile tile_cell_num_.resize(attribute_num_+1); for(int i=0; iconfig()->get_filesystem(); init_file_buffers(); // Intialize compression for tiles per attribute codec_.resize(attribute_num_+1); for(int i=0; ivar_size(i)) { offsets_codec_[i] = Codec::create(array_schema_, i, true); } else { offsets_codec_[i] = NULL; } } } WriteState::~WriteState() { // Delete codec instances for(auto i=0u; iarray()->array_schema(); int attribute_num = array_schema->attribute_num(); // Write last tile (applicable only to the sparse case) if(tile_cell_num_[attribute_num] != 0) { if(write_last_tile() != TILEDB_WS_OK) return TILEDB_WS_ERR; tile_cell_num_[attribute_num] = 0; } if (write_file_buffers() != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Sync all attributes if(sync() != TILEDB_WS_OK) return TILEDB_WS_ERR; // Success return TILEDB_WS_OK; } int WriteState::sync() { // No sync needed for storage classes derived from StorageCloudFS if (dynamic_cast(fs_)) { return TILEDB_FS_OK; } // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); const std::vector& attribute_ids = fragment_->array()->attribute_ids(); int write_method = fragment_->array()->config()->write_method(); #ifdef HAVE_MPI MPI_Comm* mpi_comm = fragment_->array()->config()->mpi_comm(); #endif std::string filename; int rc; // Sync all attributes for(int i=0; i<(int)attribute_ids.size(); ++i) { // For all attributes filename = fragment_->fragment_name() + "/" + array_schema->attribute(attribute_ids[i]) + TILEDB_FILE_SUFFIX; if(write_method == TILEDB_IO_WRITE) { rc = ::sync_path(fs_, filename); // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_sync(mpi_comm, filename.c_str()); // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } #else // Error: MPI not supported std::string errmsg = "Cannot sync; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } else { assert(0); } // Only for variable-size attributes (they have an extra file) if(array_schema->var_size(attribute_ids[i])) { filename = fragment_->fragment_name() + "/" + array_schema->attribute(attribute_ids[i]) + "_var" + TILEDB_FILE_SUFFIX; if(write_method == TILEDB_IO_WRITE) { rc = ::sync_path(fs_, filename); // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_sync(mpi_comm, filename.c_str()); // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } #else // Error: MPI not supported std::string errmsg = "Cannot sync; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } else { assert(0); } } // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } } // Sync fragment directory filename = fragment_->fragment_name(); if(write_method == TILEDB_IO_WRITE) { rc = ::sync_path(fs_, filename); } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_sync(mpi_comm, filename.c_str()); #else // Error: MPI not supported std::string errmsg = "Cannot sync; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } else { assert(0); } // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } // Success return TILEDB_WS_OK; } int WriteState::sync_attribute(const std::string& attribute) { // No sync needed for storage classes derived from StorageCloudFS if (dynamic_cast(fs_)) { return TILEDB_FS_OK; } // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int write_method = fragment_->array()->config()->write_method(); #ifdef HAVE_MPI MPI_Comm* mpi_comm = fragment_->array()->config()->mpi_comm(); #endif int attribute_id = array_schema->attribute_id(attribute); std::string filename; int rc = TILEDB_OK; // Sync attribute filename = fragment_->fragment_name() + "/" + attribute + TILEDB_FILE_SUFFIX; if(write_method == TILEDB_IO_WRITE) { rc = ::sync_path(fs_, filename); } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_sync(mpi_comm, filename.c_str()); #else // Error: MPI not supported std::string errmsg = "Cannot sync attribute; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } else { assert(0); } // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } // Only for variable-size attributes (they have an extra file) if(array_schema->var_size(attribute_id)) { filename = fragment_->fragment_name() + "/" + attribute + "_var" + TILEDB_FILE_SUFFIX; if(write_method == TILEDB_IO_WRITE) { rc = ::sync_path(fs_, filename); } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_sync(mpi_comm, filename.c_str()); #else // Error: MPI not supported std::string errmsg = "Cannot sync attribute; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } else { assert(0); } } // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } // Sync fragment directory filename = fragment_->fragment_name(); if(write_method == TILEDB_IO_WRITE) { rc = ::sync_path(fs_, filename); } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_sync(mpi_comm, filename.c_str()); #else // Error: MPI not supported std::string errmsg = "Cannot sync attribute; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } else { assert(0); } // Handle error if(rc != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } // Success return TILEDB_WS_OK; } void WriteState::init_file_buffers() { file_buffer_.resize(attribute_num_+1); file_var_buffer_.resize(attribute_num_+1); for(int i=0; ifragment_name() + "/" + TILEDB_COORDS + TILEDB_FILE_SUFFIX; } else { filename = fragment_->fragment_name() + "/" + array_schema_->attribute(attribute_id) + (is_var?"_var":"") +TILEDB_FILE_SUFFIX; } return filename; } int WriteState::write_file_buffers() { int rc = TILEDB_WS_OK; for(int i=0; ifinalize() || rc; delete file_buffer_[i]; file_buffer_[i] = NULL; } else { rc = close_file(fs_, construct_filename(i, false)) || rc; } if (file_var_buffer_[i] != NULL) { rc = file_var_buffer_[i]->finalize() || rc; delete file_var_buffer_[i]; file_var_buffer_[i] = NULL; } else { rc = close_file(fs_, construct_filename(i, true)) || rc; } if (rc) { std::string errmsg = "Could not finalize files from storage buffers for attribute " + fragment_->fragment_name() + construct_filename(i, false); PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } // For variable length attributes, ensure an empty file exists even if there // are no valid values for querying. if(!rc && array_schema_->var_size(i) && is_file(fs_, construct_filename(i, false))) { std::string filename = construct_filename(i, true); if (!is_file(fs_, filename)) { rc = create_file(fs_, filename.c_str(), O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR; if (rc) { std::string errmsg = "Cannot create file " + filename; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; rc = TILEDB_WS_ERR; } } } } return rc; } int WriteState::write_segment(int attribute_id, bool is_var, const void *segment, size_t length) { // Construct the attribute file name std::string filename = construct_filename(attribute_id, is_var); // Use buffers when desired, otherwise rely on HDFS or the Posix Filesystem to handle buffering if (fs_->get_upload_buffer_size() > 0) { StorageBuffer *file_buffer; if (is_var) { assert((attribute_id < attribute_num_) && "Coords attribute cannot be variable"); if (file_var_buffer_[attribute_id] == NULL) { file_var_buffer_[attribute_id]= new StorageBuffer(fs_, filename, fs_->get_upload_buffer_size()); } file_buffer = file_var_buffer_[attribute_id]; } else { if (file_buffer_[attribute_id] == NULL) { file_buffer_[attribute_id] = new StorageBuffer(fs_, filename, fs_->get_upload_buffer_size()); } file_buffer = file_buffer_[attribute_id]; } // Buffered writing to help with distributed filesystem and cloud performance if (file_buffer != NULL) { if (file_buffer->append_buffer(segment, length) == TILEDB_BF_ERR) { std::string errmsg = "Cannot write attribute file " + filename + " to memory buffer. Will try write directly to file"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; } else { return TILEDB_WS_OK; } } } // Write_segment directly int rc = TILEDB_WS_OK; int write_method = array_->config()->write_method(); if(write_method == TILEDB_IO_WRITE) { rc = write_to_file(fs_, filename.c_str(), segment, length); } else if(write_method == TILEDB_IO_MPI) { #ifdef HAVE_MPI rc = mpi_io_write_to_file(array_->config()->mpi_comm(), filename.c_str(), segment, length); #else // Error: MPI not supported std::string errmsg = "Cannot write segment to file; MPI not supported"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; #endif } if (rc != TILEDB_UT_OK) { std::string errmsg = "Cannot write segment to file"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg + '\n' + tiledb_ut_errmsg; return TILEDB_WS_ERR; } return TILEDB_WS_OK; } int WriteState::write(const void** buffers, const size_t* buffer_sizes) { // Create fragment directory if it does not exist std::string fragment_name = fragment_->fragment_name(); if(!is_dir(fs_, fragment_name)) { if(create_dir(fs_, fragment_name) != TILEDB_UT_OK) { tiledb_ws_errmsg = tiledb_ut_errmsg; return TILEDB_WS_ERR; } } // Dispatch the proper write command if(fragment_->mode() == TILEDB_ARRAY_WRITE || fragment_->mode() == TILEDB_ARRAY_WRITE_SORTED_COL || fragment_->mode() == TILEDB_ARRAY_WRITE_SORTED_ROW) { // SORTED if(fragment_->dense()) // DENSE FRAGMENT return write_dense(buffers, buffer_sizes); else // SPARSE FRAGMENT return write_sparse(buffers, buffer_sizes); } else if (fragment_->mode() == TILEDB_ARRAY_WRITE_UNSORTED) { // UNSORTED return write_sparse_unsorted(buffers, buffer_sizes); } else { std::string errmsg = "Cannot write to fragment; Invalid mode"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ int WriteState::compress_tile( int attribute_id, unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size, bool compress_offsets) { Codec* codec; if (compress_offsets) { codec = offsets_codec_[attribute_id]; if (codec == NULL) { tile_compressed = reinterpret_cast(&tile); tile_compressed_size = tile_size; return TILEDB_WS_OK; } } else { codec = codec_[attribute_id]; } if(codec->compress_tile(tile, tile_size, tile_compressed, tile_compressed_size) != TILEDB_CD_OK) { std::string errmsg = "Cannot compress tile for " + construct_filename(attribute_id, compress_offsets); PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } return TILEDB_WS_OK; } int WriteState::compress_and_write_tile(int attribute_id) { // For easy reference unsigned char* tile = static_cast(tiles_[attribute_id]); size_t tile_size = tile_offsets_[attribute_id]; // Trivial case - No in-memory tile if(tile_size == 0) return TILEDB_WS_OK; // Compress tile void *tile_compressed; size_t tile_compressed_size; if(compress_tile( attribute_id, tile, tile_size, &tile_compressed, tile_compressed_size, array_schema_->var_size(attribute_id)) != TILEDB_WS_OK) return TILEDB_WS_ERR; // Write segment to file int rc = write_segment(attribute_id, false, tile_compressed, tile_compressed_size); // Error if(rc != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Append offset to book-keeping book_keeping_->append_tile_offset(attribute_id, tile_compressed_size); // Success return TILEDB_WS_OK; } int WriteState::compress_and_write_tile_var(int attribute_id) { // For easy reference unsigned char* tile = static_cast(tiles_var_[attribute_id]); size_t tile_size = tiles_var_offsets_[attribute_id]; // Trivial case - No in-memory tile if(tile_size == 0) { // Append offset to book-keeping book_keeping_->append_tile_var_offset(attribute_id, 0u); book_keeping_->append_tile_var_size(attribute_id, 0u); return TILEDB_WS_OK; } // Compress tile void *tile_compressed; size_t tile_compressed_size; if(compress_tile( attribute_id, tile, tile_size, &tile_compressed, tile_compressed_size) != TILEDB_WS_OK) return TILEDB_WS_ERR; // Write segment to file int rc = write_segment(attribute_id, true, tile_compressed, tile_compressed_size); // Error if(rc != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Append offset to book-keeping book_keeping_->append_tile_var_offset(attribute_id, tile_compressed_size); book_keeping_->append_tile_var_size(attribute_id, tile_size); // Success return TILEDB_WS_OK; } template void WriteState::expand_mbr(const T* coords) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int attribute_num = array_schema->attribute_num(); int dim_num = array_schema->dim_num(); T* mbr = static_cast(mbr_); // Initialize MBR if(tile_cell_num_[attribute_num] == 0) { for(int i=0; i(buffer); size_t* shifted_buffer_s = static_cast(shifted_buffer); size_t& buffer_var_offset = buffer_var_offsets_[attribute_id]; // Shift offsets for(int64_t i=0; i& cell_pos) const { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int coords_type = array_schema->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) sort_cell_pos(buffer, buffer_size, cell_pos); else if(coords_type == TILEDB_INT64) sort_cell_pos(buffer, buffer_size, cell_pos); else if(coords_type == TILEDB_FLOAT32) sort_cell_pos(buffer, buffer_size, cell_pos); else if(coords_type == TILEDB_FLOAT64) sort_cell_pos(buffer, buffer_size, cell_pos); } template void WriteState::sort_cell_pos( const void* buffer, size_t buffer_size, std::vector& cell_pos) const { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int dim_num = array_schema->dim_num(); size_t coords_size = array_schema->coords_size(); int64_t buffer_cell_num = buffer_size / coords_size; int cell_order = array_schema->cell_order(); const T* buffer_T = static_cast(buffer); // Populate cell_pos cell_pos.resize(buffer_cell_num); for(int i=0; itile_extents() == NULL) { // NO TILE GRID if(cell_order == TILEDB_ROW_MAJOR) { // Sort cell positions SORT( cell_pos.begin(), cell_pos.end(), SmallerRow(buffer_T, dim_num)); } else if(cell_order == TILEDB_COL_MAJOR) { // Sort cell positions SORT( cell_pos.begin(), cell_pos.end(), SmallerCol(buffer_T, dim_num)); } else if(cell_order == TILEDB_HILBERT) { // Get hilbert ids std::vector ids; ids.resize(buffer_cell_num); for(int i=0; ihilbert_id(&buffer_T[i * dim_num]); // Sort cell positions SORT( cell_pos.begin(), cell_pos.end(), SmallerIdRow(buffer_T, dim_num, ids)); } else { assert(0); // The code should never reach here } } else { // TILE GRID // Get tile ids std::vector ids; ids.resize(buffer_cell_num); for(int i=0; itile_id(&buffer_T[i * dim_num]); // Sort cell positions if(cell_order == TILEDB_ROW_MAJOR) { SORT( cell_pos.begin(), cell_pos.end(), SmallerIdRow(buffer_T, dim_num, ids)); } else if(cell_order == TILEDB_COL_MAJOR) { SORT( cell_pos.begin(), cell_pos.end(), SmallerIdCol(buffer_T, dim_num, ids)); } else { assert(0); // The code should never reach here } } } void WriteState::update_book_keeping( const void* buffer, size_t buffer_size) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int coords_type = array_schema->coords_type(); // Invoke the proper templated function if(coords_type == TILEDB_INT32) update_book_keeping(buffer, buffer_size); else if(coords_type == TILEDB_INT64) update_book_keeping(buffer, buffer_size); else if(coords_type == TILEDB_FLOAT32) update_book_keeping(buffer, buffer_size); else if(coords_type == TILEDB_FLOAT64) update_book_keeping(buffer, buffer_size); } template void WriteState::update_book_keeping( const void* buffer, size_t buffer_size) { // Trivial case if(buffer_size == 0) return; // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int attribute_num = array_schema->attribute_num(); int dim_num = array_schema->dim_num(); int64_t capacity = array_schema->capacity(); size_t coords_size = array_schema->coords_size(); int64_t buffer_cell_num = buffer_size / coords_size; const T* buffer_T = static_cast(buffer); int64_t& tile_cell_num = tile_cell_num_[attribute_num]; // Update bounding coordinates and MBRs for(int64_t i = 0; i(bounding_coords_) + coords_size, &buffer_T[i*dim_num], coords_size); // Expand MBR expand_mbr(&buffer_T[i*dim_num]); // Advance a cell ++tile_cell_num; // Send MBR and bounding coordinates to book-keeping if(tile_cell_num == capacity) { book_keeping_->append_mbr(mbr_); book_keeping_->append_bounding_coords(bounding_coords_); tile_cell_num = 0; } } } int WriteState::write_last_tile() { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int attribute_num = array_schema->attribute_num(); // Send last MBR, bounding coordinates and tile cell number to book-keeping book_keeping_->append_mbr(mbr_); book_keeping_->append_bounding_coords(bounding_coords_); book_keeping_->set_last_tile_cell_num(tile_cell_num_[attribute_num]); // Flush the last tile for each compressed attribute (it is still in main // memory for(int i=0; icompression(i) != TILEDB_NO_COMPRESSION) { if(compress_and_write_tile(i) != TILEDB_WS_OK) return TILEDB_WS_ERR; if(array_schema->var_size(i)) { if(compress_and_write_tile_var(i) != TILEDB_WS_OK) return TILEDB_WS_ERR; } } } // Success return TILEDB_WS_OK; } int WriteState::write_dense( const void** buffers, const size_t* buffer_sizes) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); const std::vector& attribute_ids = fragment_->array()->attribute_ids(); int attribute_id_num = attribute_ids.size(); // Write each attribute individually int buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) { // FIXED CELLS if(write_dense_attr( attribute_ids[i], buffers[buffer_i], buffer_sizes[buffer_i]) != TILEDB_WS_OK) return TILEDB_WS_ERR; ++buffer_i; } else { // VARIABLE-SIZED CELLS if(write_dense_attr_var( attribute_ids[i], buffers[buffer_i], // offsets buffer_sizes[buffer_i], buffers[buffer_i+1], // actual cell values buffer_sizes[buffer_i+1]) != TILEDB_WS_OK) return TILEDB_WS_ERR; buffer_i += 2; } } // Success return TILEDB_WS_OK; } int WriteState::write_dense_attr( int attribute_id, const void* buffer, size_t buffer_size) { // Trivial case if(buffer_size == 0) return TILEDB_WS_OK; // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int compression = array_schema->compression(attribute_id); // No compression if(compression == TILEDB_NO_COMPRESSION) return write_dense_attr_cmp_none(attribute_id, buffer, buffer_size); else // All compressions return write_dense_attr_cmp(attribute_id, buffer, buffer_size); } int WriteState::write_dense_attr_cmp_none( int attribute_id, const void* buffer, size_t buffer_size) { // Write buffer to file return write_segment(attribute_id, false, buffer, buffer_size); } int WriteState::write_dense_attr_cmp( int attribute_id, const void* buffer, size_t buffer_size) { // For easy reference size_t tile_size = fragment_->tile_size(attribute_id); // Initialize local tile buffer if needed if(tiles_[attribute_id] == NULL) tiles_[attribute_id] = malloc(tile_size); // For easy reference char* tile = static_cast(tiles_[attribute_id]); size_t& tile_offset = tile_offsets_[attribute_id]; const char* buffer_c = static_cast(buffer); size_t buffer_offset = 0; // Bytes to fill the potentially partially buffered tile size_t bytes_to_fill = tile_size - tile_offset; // The buffer has enough cells to fill at least one tile if(bytes_to_fill <= buffer_size) { // Fill up current tile memcpy( tile + tile_offset, buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; tile_offset += bytes_to_fill; // Compress current tile and write it to disk if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) return TILEDB_WS_ERR; // Update local tile buffer offset tile_offset = 0; } // Continue to fill and compress entire tiles while(buffer_offset + tile_size <= buffer_size) { // Prepare tile memcpy(tile, buffer_c + buffer_offset, tile_size); buffer_offset += tile_size; tile_offset += tile_size; // Compress and write current tile if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) return TILEDB_WS_ERR; // Update local tile buffer offset tile_offset = 0; } // Partially fill the (new) current tile bytes_to_fill = buffer_size - buffer_offset; if(bytes_to_fill != 0) { memcpy(tile + tile_offset, buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; assert(buffer_offset == buffer_size); tile_offset += bytes_to_fill; } // Success return TILEDB_WS_OK; } int WriteState::write_dense_attr_var( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size) { // Trivial case if(buffer_size == 0) return TILEDB_WS_OK; // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int compression = array_schema->compression(attribute_id); // No compression if(compression == TILEDB_NO_COMPRESSION) return write_dense_attr_var_cmp_none( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size); else // All compressions return write_dense_attr_var_cmp( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size); // Sanity check assert(0); // Error return TILEDB_WS_ERR; } int WriteState::write_dense_attr_var_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size) { int rc = write_segment(attribute_id, true, buffer_var, buffer_var_size); // Error if(rc != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Recalculate offsets void* shifted_buffer = malloc(buffer_size); shift_var_offsets( attribute_id, buffer_var_size, buffer, buffer_size, shifted_buffer); rc = write_segment(attribute_id, false, shifted_buffer, buffer_size); // Clean up free(shifted_buffer); // Error if(rc != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Success return TILEDB_WS_OK; } int WriteState::write_dense_attr_var_cmp( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; int64_t cell_num_per_tile = fragment_->cell_num_per_tile(); size_t tile_size = cell_num_per_tile * cell_size; // Initialize local tile buffer if needed if(tiles_[attribute_id] == NULL) tiles_[attribute_id] = malloc(tile_size); // Initialize local variable tile buffer if needed if(tiles_var_[attribute_id] == NULL) { tiles_var_[attribute_id] = malloc(tile_size); tiles_var_sizes_[attribute_id] = tile_size; } // Recalculate offsets void* shifted_buffer = malloc(buffer_size); shift_var_offsets( attribute_id, buffer_var_size, buffer, buffer_size, shifted_buffer); // For easy reference char* tile = static_cast(tiles_[attribute_id]); size_t& tile_offset = tile_offsets_[attribute_id]; const size_t* buffer_s = static_cast(buffer); const char* shifted_buffer_c = static_cast(shifted_buffer); size_t buffer_offset = 0; char* tile_var = static_cast(tiles_var_[attribute_id]); size_t& tile_var_offset = tiles_var_offsets_[attribute_id]; const char* buffer_var_c = static_cast(buffer_var); size_t buffer_var_offset = 0; // Update total number of cells int64_t buffer_cell_num = buffer_size / cell_size; // Bytes to fill the potentially partially buffered tile size_t bytes_to_fill = tile_size - tile_offset; int64_t cell_num_to_fill = bytes_to_fill / cell_size; int64_t end_cell_pos = cell_num_to_fill; size_t bytes_to_fill_var = (end_cell_pos == buffer_cell_num) ? buffer_var_size : buffer_s[end_cell_pos]; // The buffer has enough cells to fill at least one tile if(bytes_to_fill <= buffer_size) { // Fill up current tile memcpy( tile + tile_offset, shifted_buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; tile_offset += bytes_to_fill; // Compress current tile and write it to disk if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_offset = 0; // Potentially expand the variable tile buffer if(tile_var_offset + bytes_to_fill_var > tiles_var_sizes_[attribute_id]) { tiles_var_sizes_[attribute_id] = tile_var_offset + bytes_to_fill_var; tiles_var_[attribute_id] = realloc(tiles_var_[attribute_id], tiles_var_sizes_[attribute_id]); // Re-allocation may assign tiles_var_ to a different region of memory tile_var = static_cast(tiles_var_[attribute_id]); } // Fill up current variable tile memcpy( tile_var + tile_var_offset, buffer_var_c + buffer_var_offset, bytes_to_fill_var); buffer_var_offset += bytes_to_fill_var; tile_var_offset += bytes_to_fill_var; // Compress current tile and write it to disk if(compress_and_write_tile_var(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_var_offset = 0; } // Continue to fill and compress entire tiles while(buffer_offset + tile_size <= buffer_size) { // Prepare tile memcpy(tile, shifted_buffer_c + buffer_offset, tile_size); buffer_offset += tile_size; tile_offset += tile_size; // Compress and write current tile if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_offset = 0; // Calculate the number of bytes to fill for the variable tile bytes_to_fill_var = (end_cell_pos + cell_num_per_tile == buffer_cell_num) ? buffer_var_size - buffer_var_offset : buffer_s[end_cell_pos + cell_num_per_tile] - buffer_s[end_cell_pos]; end_cell_pos += cell_num_per_tile; // Potentially expand the variable tile buffer if(tile_var_offset + bytes_to_fill_var > tiles_var_sizes_[attribute_id]) { tiles_var_sizes_[attribute_id] = tile_var_offset + bytes_to_fill_var; tiles_var_[attribute_id] = realloc(tiles_var_[attribute_id], tiles_var_sizes_[attribute_id]); // Re-allocation may assign tiles_var_ to a different region of memory tile_var = static_cast(tiles_var_[attribute_id]); } // Fill up current variable tile memcpy( tile_var + tile_var_offset, buffer_var_c + buffer_var_offset, bytes_to_fill_var); buffer_var_offset += bytes_to_fill_var; tile_var_offset += bytes_to_fill_var; // Compress current tile and write it to disk if(compress_and_write_tile_var(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_var_offset = 0; } // Partially fill the (new) current tile bytes_to_fill = buffer_size - buffer_offset; if(bytes_to_fill != 0) { memcpy(tile + tile_offset, shifted_buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; assert(buffer_offset == buffer_size); tile_offset += bytes_to_fill; // Calculate the number of bytes to fill for the variable tile bytes_to_fill_var = buffer_var_size - buffer_var_offset; // Potentially expand the variable tile buffer if(tile_var_offset + bytes_to_fill_var > tiles_var_sizes_[attribute_id]) { tiles_var_sizes_[attribute_id] = tile_var_offset + bytes_to_fill_var; tiles_var_[attribute_id] = realloc(tiles_var_[attribute_id], tiles_var_sizes_[attribute_id]); // Re-allocation may assign tiles_var_ to a different region of memory tile_var = static_cast(tiles_var_[attribute_id]); } // Fill up current variable tile memcpy( tile_var + tile_var_offset, buffer_var_c + buffer_var_offset, bytes_to_fill_var); buffer_var_offset += bytes_to_fill_var; assert(buffer_var_offset == buffer_var_size); tile_var_offset += bytes_to_fill_var; } // Clean up free(shifted_buffer); // Success return TILEDB_WS_OK; } int WriteState::write_sparse( const void** buffers, const size_t* buffer_sizes) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); const std::vector& attribute_ids = fragment_->array()->attribute_ids(); int attribute_id_num = attribute_ids.size(); // Write each attribute individually int buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) { // FIXED CELLS if(write_sparse_attr( attribute_ids[i], buffers[buffer_i], buffer_sizes[buffer_i]) != TILEDB_WS_OK) return TILEDB_WS_ERR; ++buffer_i; } else { // VARIABLE-SIZED CELLS if(write_sparse_attr_var( attribute_ids[i], buffers[buffer_i], // offsets buffer_sizes[buffer_i], buffers[buffer_i+1], // actual cell values buffer_sizes[buffer_i+1]) != TILEDB_WS_OK) return TILEDB_WS_ERR; buffer_i += 2; } } // Success return TILEDB_WS_OK; } int WriteState::write_sparse_attr( int attribute_id, const void* buffer, size_t buffer_size) { // Trivial case if(buffer_size == 0) return TILEDB_WS_OK; // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int compression = array_schema->compression(attribute_id); // No compression if(compression == TILEDB_NO_COMPRESSION) return write_sparse_attr_cmp_none(attribute_id, buffer, buffer_size); else // All compressions return write_sparse_attr_cmp(attribute_id, buffer, buffer_size); } int WriteState::write_sparse_attr_cmp_none( int attribute_id, const void* buffer, size_t buffer_size) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int attribute_num = array_schema->attribute_num(); // Update book-keeping if(attribute_id == attribute_num) update_book_keeping(buffer, buffer_size); // Write buffer to file int rc = write_segment(attribute_id, false, buffer, buffer_size); // Error if(rc != TILEDB_UT_OK) { return TILEDB_WS_ERR; } // Success return TILEDB_WS_OK; } int WriteState::write_sparse_attr_cmp( int attribute_id, const void* buffer, size_t buffer_size) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int attribute_num = array_schema->attribute_num(); size_t tile_size = fragment_->tile_size(attribute_id); // Update book-keeping if(attribute_id == attribute_num) update_book_keeping(buffer, buffer_size); // Initialize local tile buffer if needed if(tiles_[attribute_id] == NULL) tiles_[attribute_id] = malloc(tile_size); // For easy reference char* tile = static_cast(tiles_[attribute_id]); size_t& tile_offset = tile_offsets_[attribute_id]; const char* buffer_c = static_cast(buffer); size_t buffer_offset = 0; // Bytes to fill the potentially partially buffered tile size_t bytes_to_fill = tile_size - tile_offset; // The buffer has enough cells to fill at least one tile if(bytes_to_fill <= buffer_size) { // Fill up current tile, and append offset to book-keeping memcpy( tile + tile_offset, buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; tile_offset += bytes_to_fill; // Compress current tile and write it to disk if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) return TILEDB_WS_ERR; // Update local tile buffer offset tile_offset = 0; } // Continue to fill and compress entire tiles while(buffer_offset + tile_size <= buffer_size) { // Prepare tile memcpy(tile, buffer_c + buffer_offset, tile_size); buffer_offset += tile_size; tile_offset += tile_size; // Compress current tile, append to segment. if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) return TILEDB_WS_ERR; // Update local tile buffer offset tile_offset = 0; } // Partially fill the (new) current tile bytes_to_fill = buffer_size - buffer_offset; if(bytes_to_fill != 0) { memcpy(tile + tile_offset, buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; assert(buffer_offset == buffer_size); tile_offset += bytes_to_fill; } // Success return TILEDB_WS_OK; } int WriteState::write_sparse_attr_var( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size) { // Trivial case if(buffer_size == 0) return TILEDB_WS_OK; // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int compression = array_schema->compression(attribute_id); // No compression if(compression == TILEDB_NO_COMPRESSION) return write_sparse_attr_var_cmp_none( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size); else // All compressions return write_sparse_attr_var_cmp( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size); } int WriteState::write_sparse_attr_var_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); // Update book-keeping assert(attribute_id != array_schema->attribute_num()); // Write buffer with variable-sized cells to disk int rc = write_segment(attribute_id, true, buffer_var, buffer_var_size); // Error if(rc != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Recalculate offsets void* shifted_buffer = malloc(buffer_size); shift_var_offsets( attribute_id, buffer_var_size, buffer, buffer_size, shifted_buffer); rc = write_segment(attribute_id, false, shifted_buffer, buffer_size); // Clean up free(shifted_buffer); // Return if(rc != TILEDB_WS_OK) { return TILEDB_WS_ERR; } // Success return TILEDB_WS_OK; } int WriteState::write_sparse_attr_var_cmp( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; int64_t cell_num_per_tile = array_schema->capacity(); size_t tile_size = fragment_->tile_size(attribute_id); // Sanity check (coordinates are always fixed-sized) assert(attribute_id != array_schema->attribute_num()); // Initialize local tile buffer if needed if(tiles_[attribute_id] == NULL) tiles_[attribute_id] = malloc(tile_size); // Initialize local variable tile buffer if needed if(tiles_var_[attribute_id] == NULL) { tiles_var_[attribute_id] = malloc(tile_size); tiles_var_sizes_[attribute_id] = tile_size; } // Recalculate offsets void* shifted_buffer = malloc(buffer_size); shift_var_offsets( attribute_id, buffer_var_size, buffer, buffer_size, shifted_buffer); // For easy reference char* tile = static_cast(tiles_[attribute_id]); size_t& tile_offset = tile_offsets_[attribute_id]; const size_t* buffer_s = static_cast(buffer); const char* shifted_buffer_c = static_cast(shifted_buffer); size_t buffer_offset = 0; char* tile_var = static_cast(tiles_var_[attribute_id]); size_t& tile_var_offset = tiles_var_offsets_[attribute_id]; const char* buffer_var_c = static_cast(buffer_var); size_t buffer_var_offset = 0; // Update total number of cells int64_t buffer_cell_num = buffer_size / cell_size; // Bytes to fill the potentially partially buffered tile size_t bytes_to_fill = tile_size - tile_offset; int64_t cell_num_to_fill = bytes_to_fill / cell_size; int64_t end_cell_pos = cell_num_to_fill; size_t bytes_to_fill_var = (end_cell_pos == buffer_cell_num) ? buffer_var_size : buffer_s[end_cell_pos]; // The buffer has enough cells to fill at least one tile if(bytes_to_fill <= buffer_size) { // Fill up current tile memcpy( tile + tile_offset, shifted_buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; tile_offset += bytes_to_fill; // Compress current tile and write it to disk if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_offset = 0; // Potentially expand the variable tile buffer while(tile_var_offset + bytes_to_fill_var > tiles_var_sizes_[attribute_id]) expand_buffer(tiles_var_[attribute_id], tiles_var_sizes_[attribute_id]); // Re-allocation may assign tiles_var_ to a different region of memory tile_var = static_cast(tiles_var_[attribute_id]); // Fill up current variable tile memcpy( tile_var + tile_var_offset, buffer_var_c + buffer_var_offset, bytes_to_fill_var); buffer_var_offset += bytes_to_fill_var; tile_var_offset += bytes_to_fill_var; // Compress current tile and write it to disk if(compress_and_write_tile_var(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_var_offset = 0; } // Continue to fill and compress entire tiles while(buffer_offset + tile_size <= buffer_size) { // Prepare tile memcpy(tile, shifted_buffer_c + buffer_offset, tile_size); buffer_offset += tile_size; tile_offset += tile_size; // Compress and write current tile if(compress_and_write_tile(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_offset = 0; // Calculate the number of bytes to fill for the variable tile bytes_to_fill_var = (end_cell_pos + cell_num_per_tile == buffer_cell_num) ? buffer_var_size - buffer_var_offset : buffer_s[end_cell_pos + cell_num_per_tile] - buffer_s[end_cell_pos]; end_cell_pos += cell_num_per_tile; // Potentially expand the variable tile buffer if(tile_var_offset + bytes_to_fill_var > tiles_var_sizes_[attribute_id]) { tiles_var_sizes_[attribute_id] = tile_var_offset + bytes_to_fill_var; tiles_var_[attribute_id] = realloc(tiles_var_[attribute_id], tiles_var_sizes_[attribute_id]); // Re-allocation may assign tiles_var_ to a different region of memory tile_var = static_cast(tiles_var_[attribute_id]); } // Fill up current variable tile memcpy( tile_var + tile_var_offset, buffer_var_c + buffer_var_offset, bytes_to_fill_var); buffer_var_offset += bytes_to_fill_var; tile_var_offset += bytes_to_fill_var; // Compress current tile and write it to disk if(compress_and_write_tile_var(attribute_id) != TILEDB_WS_OK) { free(shifted_buffer); return TILEDB_WS_ERR; } // Update local tile buffer offset tile_var_offset = 0; } // Partially fill the (new) current tile bytes_to_fill = buffer_size - buffer_offset; if(bytes_to_fill != 0) { memcpy(tile + tile_offset, shifted_buffer_c + buffer_offset, bytes_to_fill); buffer_offset += bytes_to_fill; assert(buffer_offset == buffer_size); tile_offset += bytes_to_fill; // Calculate the number of bytes to fill for the variable tile bytes_to_fill_var = buffer_var_size - buffer_var_offset; // Potentially expand the variable tile buffer if(tile_var_offset + bytes_to_fill_var > tiles_var_sizes_[attribute_id]) { tiles_var_sizes_[attribute_id] = tile_var_offset + bytes_to_fill_var; tiles_var_[attribute_id] = realloc(tiles_var_[attribute_id], tiles_var_sizes_[attribute_id]); // Re-allocation may assign tiles_var_ to a different region of memory tile_var = static_cast(tiles_var_[attribute_id]); } // Fill up current variable tile memcpy( tile_var + tile_var_offset, buffer_var_c + buffer_var_offset, bytes_to_fill_var); buffer_var_offset += bytes_to_fill_var; assert(buffer_var_offset == buffer_var_size); tile_var_offset += bytes_to_fill_var; } // Clean up free(shifted_buffer); // Success return TILEDB_WS_OK; } int WriteState::write_sparse_unsorted( const void** buffers, const size_t* buffer_sizes) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int attribute_num = array_schema->attribute_num(); const std::vector& attribute_ids = fragment_->array()->attribute_ids(); int attribute_id_num = attribute_ids.size(); // Find the coordinates buffer int coords_buffer_i = -1; int buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) // FIXED CELLS ++buffer_i; else // VARIABLE-SIZED CELLS buffer_i +=2; } // Coordinates are missing if(coords_buffer_i == -1) { std::string errmsg = "Cannot write sparse unsorted; Coordinates missing"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } // Sort cell positions std::vector cell_pos; sort_cell_pos( buffers[coords_buffer_i], buffer_sizes[coords_buffer_i], cell_pos); // Write each attribute individually buffer_i=0; for(int i=0; ivar_size(attribute_ids[i])) { // FIXED CELLS if(write_sparse_unsorted_attr( attribute_ids[i], buffers[buffer_i], buffer_sizes[buffer_i], cell_pos) != TILEDB_WS_OK) return TILEDB_WS_ERR; ++buffer_i; } else { // VARIABLE-SIZED CELLS if(write_sparse_unsorted_attr_var( attribute_ids[i], buffers[buffer_i], // offsets buffer_sizes[buffer_i], buffers[buffer_i+1], // actual values buffer_sizes[buffer_i+1], cell_pos) != TILEDB_WS_OK) return TILEDB_WS_ERR; buffer_i += 2; } } // Success return TILEDB_WS_OK; } int WriteState::write_sparse_unsorted_attr( int attribute_id, const void* buffer, size_t buffer_size, const std::vector& cell_pos) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int compression = array_schema->compression(attribute_id); // No compression if(compression == TILEDB_NO_COMPRESSION) return write_sparse_unsorted_attr_cmp_none( attribute_id, buffer, buffer_size, cell_pos); else // All compressions return write_sparse_unsorted_attr_cmp( attribute_id, buffer, buffer_size, cell_pos); } int WriteState::write_sparse_unsorted_attr_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const std::vector& cell_pos) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); size_t cell_size = array_schema->cell_size(attribute_id); const char* buffer_c = static_cast(buffer); // Check number of cells in buffer int64_t buffer_cell_num = buffer_size / cell_size; if(buffer_cell_num != int64_t(cell_pos.size())) { std::string errmsg = std::string("Cannot write sparse unsorted; Invalid number of " "cells in attribute '") + array_schema->attribute(attribute_id) + "'"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } // Allocate a local buffer to hold the sorted cells char* sorted_buffer = new char[TILEDB_SORTED_BUFFER_SIZE]; size_t sorted_buffer_size = 0; // Sort and write attribute values in batches for(int64_t i=0; i TILEDB_SORTED_BUFFER_SIZE) { if(write_sparse_attr_cmp_none( attribute_id, sorted_buffer, sorted_buffer_size) != TILEDB_WS_OK) { delete [] sorted_buffer; return TILEDB_WS_ERR; } else { sorted_buffer_size = 0; } } // Keep on copying the cells in the sorted order in the sorted buffer memcpy( sorted_buffer + sorted_buffer_size, buffer_c + cell_pos[i] * cell_size, cell_size); sorted_buffer_size += cell_size; } // Write final batch if(sorted_buffer_size != 0) { if(write_sparse_attr_cmp_none( attribute_id, sorted_buffer, sorted_buffer_size) != TILEDB_WS_OK) { delete [] sorted_buffer; return TILEDB_WS_ERR; } } // Clean up delete [] sorted_buffer; // Success return TILEDB_WS_OK; } int WriteState::write_sparse_unsorted_attr_cmp( int attribute_id, const void* buffer, size_t buffer_size, const std::vector& cell_pos) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); size_t cell_size = array_schema->cell_size(attribute_id); const char* buffer_c = static_cast(buffer); // Check number of cells in buffer int64_t buffer_cell_num = buffer_size / cell_size; if(buffer_cell_num != int64_t(cell_pos.size())) { std::string errmsg = std::string("Cannot write sparse unsorted; Invalid number of " "cells in attribute '") + array_schema->attribute(attribute_id) + "'"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } // Allocate a local buffer to hold the sorted cells char* sorted_buffer = new char[TILEDB_SORTED_BUFFER_SIZE]; size_t sorted_buffer_size = 0; // Sort and write attribute values in batches for(int64_t i=0; i TILEDB_SORTED_BUFFER_SIZE) { if(write_sparse_attr_cmp( attribute_id, sorted_buffer, sorted_buffer_size) != TILEDB_WS_OK) { delete [] sorted_buffer; return TILEDB_WS_ERR; } else { sorted_buffer_size = 0; } } // Keep on copying the cells in the sorted order in the sorted buffer memcpy( sorted_buffer + sorted_buffer_size, buffer_c + cell_pos[i] * cell_size, cell_size); sorted_buffer_size += cell_size; } // Write final batch if(sorted_buffer_size != 0) { if(write_sparse_attr_cmp( attribute_id, sorted_buffer, sorted_buffer_size) != TILEDB_WS_OK) { delete [] sorted_buffer; return TILEDB_WS_ERR; } } // Clean up delete [] sorted_buffer; // Success return TILEDB_WS_OK; } int WriteState::write_sparse_unsorted_attr_var( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size, const std::vector& cell_pos) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int compression = array_schema->compression(attribute_id); // No compression if(compression == TILEDB_NO_COMPRESSION) return write_sparse_unsorted_attr_var_cmp_none( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size, cell_pos); else // All compressions return write_sparse_unsorted_attr_var_cmp( attribute_id, buffer, buffer_size, buffer_var, buffer_var_size, cell_pos); } int WriteState::write_sparse_unsorted_attr_var_cmp_none( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size, const std::vector& cell_pos) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; size_t cell_var_size; const size_t* buffer_s = static_cast(buffer); const char* buffer_var_c = static_cast(buffer_var); // Check number of cells in buffer int64_t buffer_cell_num = buffer_size / cell_size; if(buffer_cell_num != int64_t(cell_pos.size())) { std::string errmsg = std::string("Cannot write sparse unsorted variable; " "Invalid number of cells in attribute '") + array_schema->attribute(attribute_id) + "'"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } // Allocate a local buffer to hold the sorted cells char* sorted_buffer = new char[TILEDB_SORTED_BUFFER_SIZE]; size_t sorted_buffer_size = 0; char* sorted_buffer_var = new char[TILEDB_SORTED_BUFFER_VAR_SIZE]; size_t sorted_buffer_var_size = 0; // Sort and write attribute values in batches for(int64_t i=0; i TILEDB_SORTED_BUFFER_SIZE || sorted_buffer_var_size + cell_var_size > TILEDB_SORTED_BUFFER_VAR_SIZE) { if(write_sparse_attr_var_cmp_none( attribute_id, sorted_buffer, sorted_buffer_size, sorted_buffer_var, sorted_buffer_var_size) != TILEDB_WS_OK) { delete [] sorted_buffer; delete [] sorted_buffer_var; return TILEDB_WS_ERR; } sorted_buffer_size = 0; sorted_buffer_var_size = 0; } // Keep on copying the cells in sorted order in the sorted buffer memcpy( sorted_buffer + sorted_buffer_size, &sorted_buffer_var_size, cell_size); sorted_buffer_size += cell_size; // Keep on copying the variable cells in sorted order in the sorted buffer memcpy( sorted_buffer_var + sorted_buffer_var_size, buffer_var_c + buffer_s[cell_pos[i]], cell_var_size); sorted_buffer_var_size += cell_var_size; } // Write final batch if(sorted_buffer_size != 0) { if(write_sparse_attr_var_cmp_none( attribute_id, sorted_buffer, sorted_buffer_size, sorted_buffer_var, sorted_buffer_var_size) != TILEDB_WS_OK) { delete [] sorted_buffer; delete [] sorted_buffer_var; return TILEDB_WS_ERR; } } // Clean up delete [] sorted_buffer; delete [] sorted_buffer_var; // Success return TILEDB_WS_OK; } int WriteState::write_sparse_unsorted_attr_var_cmp( int attribute_id, const void* buffer, size_t buffer_size, const void* buffer_var, size_t buffer_var_size, const std::vector& cell_pos) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; size_t cell_var_size; const size_t* buffer_s = static_cast(buffer); const char* buffer_var_c = static_cast(buffer_var); // Check number of cells in buffer int64_t buffer_cell_num = buffer_size / cell_size; if(buffer_cell_num != int64_t(cell_pos.size())) { std::string errmsg = std::string("Cannot write sparse unsorted variable; " "Invalid number of cells in attribute '") + array_schema->attribute(attribute_id) + "'"; PRINT_ERROR(errmsg); tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; return TILEDB_WS_ERR; } // Allocate a local buffer to hold the sorted cells char* sorted_buffer = new char[TILEDB_SORTED_BUFFER_SIZE]; size_t sorted_buffer_size = 0; char* sorted_buffer_var = new char[TILEDB_SORTED_BUFFER_VAR_SIZE]; size_t sorted_buffer_var_size = 0; // Sort and write attribute values in batches for(int64_t i=0; i TILEDB_SORTED_BUFFER_SIZE || sorted_buffer_var_size + cell_var_size > TILEDB_SORTED_BUFFER_VAR_SIZE) { if(write_sparse_attr_var_cmp( attribute_id, sorted_buffer, sorted_buffer_size, sorted_buffer_var, sorted_buffer_var_size) != TILEDB_WS_OK) { delete [] sorted_buffer; delete [] sorted_buffer_var; return TILEDB_WS_ERR; } sorted_buffer_size = 0; sorted_buffer_var_size = 0; } // Keep on copying the cells in sorted order in the sorted buffer memcpy( sorted_buffer + sorted_buffer_size, &sorted_buffer_var_size, cell_size); sorted_buffer_size += cell_size; // Keep on copying the variable cells in sorted order in the sorted buffer memcpy( sorted_buffer_var + sorted_buffer_var_size, buffer_var_c + buffer_s[cell_pos[i]], cell_var_size); sorted_buffer_var_size += cell_var_size; } // Write final batch if(sorted_buffer_size != 0) { if(write_sparse_attr_var_cmp( attribute_id, sorted_buffer, sorted_buffer_size, sorted_buffer_var, sorted_buffer_var_size) != TILEDB_WS_OK) { delete [] sorted_buffer; delete [] sorted_buffer_var; return TILEDB_WS_ERR; } } // Clean up delete [] sorted_buffer; delete [] sorted_buffer_var; // Success return TILEDB_WS_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/metadata/000077500000000000000000000000001453617025200216445ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/metadata/metadata.cc000066400000000000000000000334251453617025200237420ustar00rootroot00000000000000/** * @file metadata.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the Metadata class. */ #include "metadata.h" #include #include #include "tiledb_openssl_shim.h" /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_MT_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_mt_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ Metadata::Metadata() { array_ = NULL; } Metadata::~Metadata() { } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ Array* Metadata::array() const { return array_; } const ArraySchema* Metadata::array_schema() const { return array_->array_schema(); } bool Metadata::overflow(int attribute_id) const { return array_->overflow(attribute_id); } int Metadata::read(const char* key, void** buffers, size_t* buffer_sizes) { // Sanity checks if(mode_ != TILEDB_METADATA_READ) { std::string errmsg = "Cannot read from metadata; Invalid mode"; PRINT_ERROR(errmsg); tiledb_mt_errmsg = TILEDB_MT_ERRMSG + errmsg; return TILEDB_MT_ERR; } // Compute subarray for the read int subarray[8]; unsigned int coords[4]; if(OpenSSL_version_num() < 0x30000000L) { MD5((const unsigned char*) key, strlen(key)+1, (unsigned char*) coords); } else { EVP_MD_CTX* mdctx; mdctx = EVP_MD_CTX_new(); EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); EVP_DigestUpdate(mdctx, key, strlen(key)+1); EVP_DigestFinal_ex(mdctx, (unsigned char*) coords, NULL); EVP_MD_CTX_free(mdctx); } for(int i=0; i<4; ++i) { subarray[2*i] = int(coords[i]); subarray[2*i+1] = int(coords[i]); } // Re-init sub array if(array_->reset_subarray(subarray) != TILEDB_AR_OK) { tiledb_mt_errmsg = tiledb_ar_errmsg; return TILEDB_MT_ERR; } // Read from array if(array_->read(buffers, buffer_sizes) != TILEDB_AR_OK) { tiledb_mt_errmsg = tiledb_ar_errmsg; return TILEDB_MT_ERR; } // Success return TILEDB_MT_OK; } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int Metadata::consolidate( Fragment*& new_fragment, std::vector& old_fragment_names) { // Consolidate if(array_->consolidate(new_fragment, old_fragment_names) != TILEDB_AR_OK) { tiledb_mt_errmsg = tiledb_ar_errmsg; return TILEDB_MT_ERR; } // Success return TILEDB_MT_OK; } int Metadata::finalize() { int rc = array_->finalize(); array_->free_array_schema(); //~Array() doesn't free schema because clone == NULL inside array_ delete array_; array_ = NULL; if(rc != TILEDB_AR_OK) { tiledb_mt_errmsg = tiledb_ar_errmsg; return TILEDB_MT_ERR; } // Success return TILEDB_MT_OK; } int Metadata::init( const ArraySchema* array_schema, const std::vector& fragment_names, const std::vector& book_keeping, int mode, const char** attributes, int attribute_num, const StorageManagerConfig* config) { // Sanity check on mode if(mode != TILEDB_METADATA_READ && mode != TILEDB_METADATA_WRITE) { std::string errmsg = "Cannot initialize metadata; Invalid metadata mode"; PRINT_ERROR(errmsg); tiledb_mt_errmsg = TILEDB_MT_ERRMSG + errmsg; return TILEDB_MT_ERR; } // Set mode mode_ = mode; int array_mode = (mode == TILEDB_METADATA_READ) ? TILEDB_ARRAY_READ : TILEDB_ARRAY_WRITE_UNSORTED; // Set attributes char** array_attributes; int array_attribute_num; if(attributes == NULL) { array_attribute_num = (mode == TILEDB_METADATA_WRITE) ? array_schema->attribute_num() + 1 : array_schema->attribute_num(); array_attributes = new char*[array_attribute_num]; for(int i=0; iattribute(i).c_str(); size_t attribute_len = strlen(attribute); array_attributes[i] = new char[attribute_len+1]; strcpy(array_attributes[i], attribute); } } else { array_attribute_num = (mode == TILEDB_METADATA_WRITE) ? attribute_num + 1 : attribute_num; array_attributes = new char*[array_attribute_num]; for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); tiledb_mt_errmsg = TILEDB_MT_ERRMSG + errmsg; return TILEDB_MT_ERR; } array_attributes[i] = new char[attribute_len+1]; strcpy(array_attributes[i], attributes[i]); } if(mode == TILEDB_METADATA_WRITE) { size_t attribute_len = strlen(TILEDB_COORDS); array_attributes[array_attribute_num] = new char[attribute_len+1]; strcpy(array_attributes[array_attribute_num], TILEDB_COORDS); } } // Initialize array array_ = new Array(); int rc = array_->init( array_schema, array_schema->array_name(), fragment_names, book_keeping, array_mode, (const char**) array_attributes, array_attribute_num, NULL, config); // Clean up for(int i=0; iarray_schema(); // Set attributes char** array_attributes; int array_attribute_num; if(attributes == NULL) { array_attribute_num = (mode_ == TILEDB_METADATA_WRITE) ? array_schema->attribute_num() + 1 : array_schema->attribute_num(); array_attributes = new char*[array_attribute_num]; for(int i=0; iattribute(i).c_str(); size_t attribute_len = strlen(attribute); array_attributes[i] = new char[attribute_len+1]; strcpy(array_attributes[i], attribute); } } else { array_attribute_num = (mode_ == TILEDB_METADATA_WRITE) ? attribute_num + 1 : attribute_num; array_attributes = new char*[array_attribute_num]; for(int i=0; i TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid attribute name length"; PRINT_ERROR(errmsg); tiledb_mt_errmsg = errmsg; return TILEDB_MT_ERR; } array_attributes[i] = new char[attribute_len+1]; strcpy(array_attributes[i], attributes[i]); } if(mode_ == TILEDB_METADATA_WRITE) { size_t attribute_len = strlen(TILEDB_COORDS); array_attributes[array_attribute_num] = new char[attribute_len+1]; strcpy(array_attributes[array_attribute_num], TILEDB_COORDS); } } // Reset attributes int rc = array_->reset_attributes( (const char**) array_attributes, array_attribute_num); // Clean up for(int i=0; iwrite(array_buffers, array_buffer_sizes); // Clean up free(coords); free(array_buffers); free(array_buffer_sizes); // Error if(rc != TILEDB_AR_OK) { tiledb_mt_errmsg = tiledb_ar_errmsg; return TILEDB_MT_ERR; } // Success return TILEDB_MT_OK; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ void Metadata::compute_array_coords( const char* keys, size_t keys_size, void*& coords, size_t& coords_size) const { // Compute keys offsets size_t* keys_offsets = (size_t*) malloc(10*sizeof(size_t)); int64_t keys_num_allocated = 10; int64_t keys_num = 0; bool null_char_found = true; for(size_t i=0; i 0); // Compute coords coords_size = keys_num * 4 * sizeof(int); coords = malloc(coords_size); size_t key_size; const unsigned char* keys_c; unsigned char* coords_c; for(int64_t i=0; iarray_schema(); int attribute_num = array_schema->attribute_num(); const std::vector attribute_ids = array_->attribute_ids(); int attribute_id_num = attribute_ids.size(); // Count number of variable-sized attributes int var_attribute_num = 0; for(int i=0; ivar_size(attribute_ids[i])) ++var_attribute_num; // Allocate space for the array buffers array_buffers = (const void**) malloc( (attribute_id_num + var_attribute_num)*sizeof(const void*)); array_buffer_sizes = (size_t*) malloc( (attribute_id_num + var_attribute_num)*sizeof(size_t)); // Set the array buffers int buffer_i = 0; int array_buffer_i = 0; for(int i=0; ivar_size(attribute_ids[i])) { // Variable-sized array_buffers[array_buffer_i] = buffers[buffer_i]; array_buffer_sizes[array_buffer_i] = buffer_sizes[buffer_i]; ++array_buffer_i; ++buffer_i; } } } } genomicsdb-0.0~git20231212.9d7ddd0/core/src/metadata/metadata_iterator.cc000066400000000000000000000070371453617025200256530ustar00rootroot00000000000000/** * @file metadata_iterator.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the MetadataIterator class. */ #include "metadata_iterator.h" /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_MIT_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_mit_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ MetadataIterator::MetadataIterator() { array_it_ = NULL; } MetadataIterator::~MetadataIterator() { } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ const std::string& MetadataIterator::metadata_name() const { return array_it_->array_name(); } bool MetadataIterator::end() const { return array_it_->end(); } int MetadataIterator::get_value( int attribute_id, const void** value, size_t* value_size) const { if(array_it_->get_value(attribute_id, value, value_size) != TILEDB_AIT_OK) { tiledb_mit_errmsg = tiledb_ait_errmsg; return TILEDB_MIT_ERR; } // Success return TILEDB_MIT_OK; } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int MetadataIterator::finalize() { int rc = array_it_->finalize(); delete array_it_; array_it_ = NULL; delete metadata_; metadata_ = NULL; // Error if(rc != TILEDB_AIT_OK) { tiledb_mit_errmsg = tiledb_ait_errmsg; return TILEDB_MIT_ERR; } // Success return TILEDB_MIT_OK; } int MetadataIterator::init( Metadata* metadata, void** buffers, size_t* buffer_sizes) { // Initialize an array iterator metadata_ = metadata; array_it_ = new ArrayIterator(); if(array_it_->init(metadata->array(), buffers, buffer_sizes) != TILEDB_AIT_OK) { delete array_it_; array_it_ = NULL; tiledb_mit_errmsg = tiledb_ait_errmsg; return TILEDB_MIT_ERR; } // Return return TILEDB_MIT_OK; } int MetadataIterator::next() { if(array_it_->next() != TILEDB_AIT_OK) { tiledb_mit_errmsg = tiledb_ait_errmsg; return TILEDB_MIT_ERR; } // Success return TILEDB_MIT_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/misc/000077500000000000000000000000001453617025200210175ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/misc/error.cc000066400000000000000000000024511453617025200224610ustar00rootroot00000000000000/** * @file error.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION Common Error Handling * */ #include "error.h" void reset_errno() { errno = 0; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/misc/hilbert_curve.cc000066400000000000000000000107041453617025200241650ustar00rootroot00000000000000/** * @file hilbert_curve.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the HilbertCurve class. */ #include "hilbert_curve.h" #include #include /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ HilbertCurve::HilbertCurve(int bits, int dim_num) : bits_(bits), dim_num_(dim_num) { assert(dim_num >=0 && dim_num < HC_MAX_DIM); assert(bits * dim_num <= int(sizeof(int64_t)*8)); } HilbertCurve::~HilbertCurve() { } /* ****************************** */ /* BASIC FUNCTIONS */ /* ****************************** */ void HilbertCurve::coords_to_hilbert(const int* coords, int64_t& hilbert) { // Copy coords to temporary storage memcpy(temp_, coords, dim_num_ * sizeof(int)); // Convert coords to the transpose form of the hilbert value AxestoTranspose(temp_, bits_, dim_num_); // Convert the hilbert transpose form into an int64_t hilbert value hilbert = 0; int64_t c = 1; // This is a bit shifted from right to left over temp_[i] int64_t h = 1; // This is a bit shifted from right to left over hilbert for(int j=0; j=0; --i, h <<= 1) { if(temp_[i] & c) hilbert |= h; } } } void HilbertCurve::hilbert_to_coords(int64_t hilbert, int* coords) { // Initialization for(int i=0; i=0; --i, h <<= 1) { if(hilbert & h) temp_[i] |= c; } } // Convert coords to the transpose form of the hilbert value TransposetoAxes(temp_, bits_, dim_num_); // Copy from the temporary storage to the (output) coords memcpy(coords, temp_, dim_num_ * sizeof(int)); } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ void HilbertCurve::AxestoTranspose(int* X, int b, int n) { int P, Q, t, i; // Inverse undo for(Q = 1 << (b - 1); Q > 1; Q >>= 1) { P = Q - 1; if(X[0] & Q) // invert X[0] ^= P; for(i = 1; i < n; i++) if(X[i] & Q) // invert X[0] ^= P; else { // exchange t = (X[0] ^ X[i]) & P; X[0] ^= t; X[i] ^= t; } } // Gray encode (inverse of decode) for(i = 1; i < n; i++) X[i] ^= X[i-1]; t = X[n-1]; for(i = 1; i < b; i <<= 1) X[n-1] ^= X[n-1] >> i; t ^= X[n-1]; for(i = n-2; i >= 0; i--) X[i] ^= t; } void HilbertCurve::TransposetoAxes(int* X, int b, int n) { int M, P, Q, t, i; // Gray decode by H ^ (H/2) t = X[n-1] >> 1; for(i = n-1; i; i--) X[i] ^= X[i-1]; X[0] ^= t; // Undo excess work M = 2 << (b - 1); for(Q = 2; Q != M; Q <<= 1) { P = Q - 1; for(i = n-1; i; i--) { if(X[i] & Q) // invert X[0] ^= P; else { // exchange t = (X[0] ^ X[i]) & P; X[0] ^= t; X[i] ^= t; } if(X[0] & Q) // invert X[0] ^= P; } } } genomicsdb-0.0~git20231212.9d7ddd0/core/src/misc/mem_utils.cc000066400000000000000000000063661453617025200233370ustar00rootroot00000000000000/** * @file mem_utils.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements utilities to help profile memory */ #include #include #include #include #include #include #ifdef __linux #include #endif typedef struct statm_t{ unsigned long size=0,resident=0,share=0,text=0,lib=0,data=0,dt=0; } statm_t; void print_time() { time_t track_time = time(NULL); tm* current = localtime(&track_time); char buffer[32]; // Format %c(locale dependent) e.g. Fri Mar 18 16:13:48 2022 for EN_US std::strftime(buffer, sizeof(buffer), "%c ", current); std::cerr << buffer; } std::string readable_size(size_t size) { std::vector suffix = { "B", "KB", "MB", "GB", "TB" }; auto i = 0u; size = size*4096; while (size > 0) { if (i > suffix.size()) break; if (size/1024 == 0) { return std::to_string(size) + suffix[i]; } else { size /= 1024; i++; } } return std::to_string(size); } void print_memory_stats(const std::string& msg) { #ifdef __linux const char* statm_path = "/proc/self/statm"; statm_t result; FILE *f = fopen(statm_path,"r"); if (!f) { perror(statm_path); abort(); } if (7 != fscanf(f,"%lu %lu %lu %lu %lu %lu %lu", &result.size, &result.resident, &result.share, &result.text, &result.lib, &result.data, &result.dt)) { perror(statm_path); abort(); } fclose(f); print_time(); std::cerr << "Memory stats " << msg << " size=" << readable_size(result.size) << " resident=" << readable_size(result.resident) << " share=" << readable_size(result.share) << " text=" << readable_size(result.text) << " lib=" << readable_size(result.lib) << " data=" << readable_size(result.data) << " dt=" << readable_size(result.dt) << std::endl; #else print_time(); std::cerr << "TBD: Memory stats " << msg << std::endl; #endif } void trim_memory() { #ifdef __linux if (malloc_trim(0)) { #ifdef DO_MEMORY_PROFILING print_memory_stats("Memory from the heap was successfully trimmed"); #endif } #endif } genomicsdb-0.0~git20231212.9d7ddd0/core/src/misc/progress_bar.cc000066400000000000000000000047011453617025200240200ustar00rootroot00000000000000/** * @file progress_bar.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements class ProgressBar. */ #include "progress_bar.h" #include /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ ProgressBar::ProgressBar(double complete, int max_length, char filler) { complete_ = complete; filler_ = filler; length_ = 0; incomplete_ = 0; last_ratio_ = 0; max_length_ = max_length; ratio_ = 0; } ProgressBar::~ProgressBar() { } /* ****************************** */ /* METHODS */ /* ****************************** */ void ProgressBar::load(double amount) { incomplete_ += amount; if(incomplete_ > complete_) incomplete_ = complete_; ratio_ = (incomplete_ / complete_); length_ = ratio_ * max_length_; // Print bar if(ratio_ - last_ratio_ > PB_RATIO_STEP) { print(); last_ratio_ = ratio_; } } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ void ProgressBar::print() { fprintf(stdout, "%3d%% [", (int)(ratio_ * 100)); for(int i=0; i #include #include #include #include #include #include #include "uri.h" #include // Constructor uri::uri(const std::string& uri_s) { parse(uri_s); } // Accessors std::string uri::protocol() { return protocol_; } std::string uri::host() { return host_; } std::string uri::port() { return port_; } int16_t uri::nport() { return nport_; } std::string uri::path() { return path_; } std::unordered_map uri::query() { return query_; } // Private Methods std::string uri::urlDecode(const std::string& uri_s) { std::string result; result.reserve(uri_s.size()); for (std::size_t i = 0; i < uri_s.size(); ++i) { auto ch = uri_s[i]; if (ch == '%' && (i + 2) < uri_s.size()) { auto hex = uri_s.substr(i + 1, 2); auto dec = static_cast(std::strtol(hex.c_str(), nullptr, 16)); result.push_back(dec); i += 2; } else { result.push_back(ch); } } return result; } void uri::parse(const std::string& uri_s) { if (uri_s.empty()) { throw std::system_error(EINVAL, std::generic_category(), "Cannot parse empty string as an URI"); } const std::string::const_iterator start_iter = uri_s.begin(); const std::string::const_iterator end_iter = uri_s.end(); const std::string protocol_end("://"); std::string::const_iterator protocol_iter = std::search(start_iter, end_iter, protocol_end.begin(), protocol_end.end()); if (protocol_iter == uri_s.end()) { throw std::system_error(EINVAL, std::generic_category(), "String does not seem to be a URI"); } // protocol is case insensitive protocol_.reserve(std::distance(start_iter, protocol_iter)); //std::transform(start_iter, protocol_iter, back_inserter(protocol_), std::ptr_fun(tolower)); std::transform(start_iter, protocol_iter, back_inserter(protocol_), [](int c){return std::tolower(c);}); if (protocol_iter == end_iter) { return; } std::advance(protocol_iter, protocol_end.length()); std::string::const_iterator path_iter = find(protocol_iter, end_iter, '/'); std::string::const_iterator port_iter = find(protocol_iter, path_iter, ':'); // host is case insensitive host_.reserve(distance(protocol_iter, port_iter)); //std::transform(protocol_iter, port_iter, back_inserter(host_), std::ptr_fun(tolower)); std::transform(protocol_iter, port_iter, back_inserter(host_), [](int c){return std::tolower(c);}); if (port_iter != path_iter) { ++port_iter; port_.assign(port_iter, path_iter); // Convert port into int16_t const char *start_ptr = port_.c_str(); char *end_ptr; errno = 0; long port_val = strtol(start_ptr, &end_ptr, 10); if (errno == ERANGE || port_val > UINT16_MAX){ throw std::system_error(ERANGE, std::generic_category(), "URI has a bad port #"); } if (start_ptr != end_ptr) { nport_ = (uint16_t)port_val; } } std::string::const_iterator query_iter = find(path_iter, end_iter, '?'); path_.assign(path_iter, query_iter); if (query_iter != end_iter) { ++query_iter; std::string queryTemp(urlDecode(std::string(query_iter, end_iter))); char* save_ptr; for (char* token = strtok_r(queryTemp.data(), "&", &save_ptr); token != NULL; token = strtok_r(NULL, "&", &save_ptr)) { char* search = strchr(token, '='); if (search == NULL || (std::size_t)(search - token) == 0) { throw std::system_error(EINVAL, std::generic_category(), "Query is in incorrect format"); } std::string key( std::string(token).substr(0, (std::size_t)(search - token))); query_[key] = std::string(++search); } } } azure_uri::azure_uri(const std::string& uri_s) : uri(uri_s) { if (this->protocol().compare("azb") == 0) { account_ = this->query()["account"]; container_ = this->host(); endpoint_ = this->query()["endpoint"]; } else { std::size_t begin = this->host().find('@'); std::size_t end = this->host().find('.'); if (begin != std::string::npos && end != std::string::npos) { account_ = this->host().substr(begin + 1, end - begin - 1); endpoint_ = this->host().substr(begin + 1, this->host().find('/', end) - begin - 1); } if (begin != std::string::npos) { container_ = this->host().substr(0, begin); } } } std::string azure_uri::account() { return account_; } std::string azure_uri::container() { return container_; } std::string azure_uri::endpoint() { return endpoint_; } s3_uri::s3_uri(const std::string& uri_s) : uri(uri_s) { bucket_ = this->host(); } std::string s3_uri::bucket() { return bucket_; } gcs_uri::gcs_uri(const std::string& uri_s) : uri(uri_s) { bucket_ = this->host(); } std::string gcs_uri::bucket() { return bucket_; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/misc/utils.cc000066400000000000000000001307601453617025200224750ustar00rootroot00000000000000/** * @file utils.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2019, 2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements useful (global) functions. */ #include "tiledb_constants.h" #include "error.h" #include "utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__APPLE__) && defined(__MACH__) #include #include #include #include #include #include #else #include #endif #include #include #include #define XSTR(s) STR(s) #define STR(s) #s /* ****************************** */ /* MACROS */ /* ****************************** */ #define UTILS_ERROR(MSG) TILEDB_ERROR(TILEDB_UT_ERRMSG, MSG, tiledb_ut_errmsg) #define UTILS_SYSTEM_ERROR(MSG) SYSTEM_ERROR(TILEDB_UT_ERRMSG, MSG, "", tiledb_ut_errmsg) #define UTILS_PATH_ERROR(MSG, PATH) SYSTEM_ERROR(TILEDB_UT_ERRMSG, MSG, PATH, tiledb_ut_errmsg) /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_ut_errmsg = ""; /* ****************************** */ /* FUNCTIONS */ /* ****************************** */ bool array_read_mode(int mode) { return mode == TILEDB_ARRAY_READ || mode == TILEDB_ARRAY_READ_SORTED_COL || mode == TILEDB_ARRAY_READ_SORTED_ROW; } bool array_write_mode(int mode) { return mode == TILEDB_ARRAY_WRITE || mode == TILEDB_ARRAY_WRITE_SORTED_COL || mode == TILEDB_ARRAY_WRITE_SORTED_ROW || mode == TILEDB_ARRAY_WRITE_UNSORTED; } bool array_consolidate_mode(int mode) { return mode == TILEDB_ARRAY_CONSOLIDATE; } bool both_slashes(char a, char b) { return a == '/' && b == '/'; } template inline bool cell_in_subarray(const T* cell, const T* subarray, int dim_num) { for(int i=0; i= subarray[2*i] && cell[i] <= subarray[2*i+1]) continue; // Inside this dimension domain return false; // NOT inside this dimension domain } return true; } template int64_t cell_num_in_subarray(const T* subarray, int dim_num) { int64_t cell_num = 1; for(int i=0; i int cmp_col_order( const T* coords_a, const T* coords_b, int dim_num) { for(int i=dim_num-1; i>=0; --i) { // a precedes b if(coords_a[i] < coords_b[i]) return -1; // b precedes a else if(coords_a[i] > coords_b[i]) return 1; } // a and b are equal return 0; } template int cmp_col_order( int64_t id_a, const T* coords_a, int64_t id_b, const T* coords_b, int dim_num) { // a precedes b if(id_a < id_b) return -1; // b precedes a if(id_a > id_b) return 1; // ids are equal, check the coordinates for(int i=dim_num-1; i>=0; --i) { // a precedes b if(coords_a[i] < coords_b[i]) return -1; // b precedes a else if(coords_a[i] > coords_b[i]) return 1; } // a and b are equal return 0; } template int cmp_row_order( const T* coords_a, const T* coords_b, int dim_num) { for(int i=0; i coords_b[i]) return 1; } // a and b are equal return 0; } template int cmp_row_order( int64_t id_a, const T* coords_a, int64_t id_b, const T* coords_b, int dim_num) { // a precedes b if(id_a < id_b) return -1; // b precedes a if(id_a > id_b) return 1; // ids are equal, check the coordinates for(int i=0; i coords_b[i]) return 1; } // a and b are equal return 0; } bool is_supported_cloud_path(const std::string& pathURL) { return is_hdfs_path(pathURL) || is_gcs_path(pathURL) || is_azure_path(pathURL) || is_azure_blob_storage_path(pathURL) || is_s3_storage_path(pathURL); } bool is_azure_path(const std::string& pathURL) { if (!pathURL.empty() && (starts_with(pathURL, "wasbs:") || starts_with(pathURL, "wasb:") || starts_with(pathURL, "abfss:") || starts_with(pathURL, "abfs") || starts_with(pathURL, "adl:"))) { return true; } else { return false; } } bool is_azure_blob_storage_path(const std::string& pathURL) { if (!pathURL.empty() && (starts_with(pathURL, "az:") || starts_with(pathURL, "azb:"))) { return true; } else { return false; } } bool is_s3_storage_path(const std::string& pathURL) { if (!pathURL.empty() && starts_with(pathURL, "s3:")) { return true; } else { return false; } } bool is_gcs_path(const std::string& pathURL) { if (!pathURL.empty() && starts_with(pathURL, "gs:")) { return true; } else { return false; } } bool is_hdfs_path(const std::string& pathURL) { if (!pathURL.empty() && (starts_with(pathURL, "hdfs:") || starts_with(pathURL, "s3a:") || starts_with(pathURL, "gs:"))) { return true; } else { return false; } } bool is_env_set(const std::string& name) { auto env_var = getenv(name.c_str()); if(env_var && ((strcasecmp(env_var, "true") == 0) || (strcmp(env_var, "1") == 0))) { return true; } else { return false; } } int create_dir(StorageFS *fs, const std::string& dir) { if (fs->create_dir(dir)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int create_file(StorageFS *fs, const std::string& filename, int flags, mode_t mode) { if (fs->create_file(filename, flags, mode)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int delete_file(StorageFS *fs, const std::string& filename) { if (fs->delete_file(filename)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int create_fragment_file(StorageFS *fs, const std::string& dir) { // Create the special fragment file std::string filename = fs->append_paths(dir, TILEDB_FRAGMENT_FILENAME); if (fs->create_file(filename, O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) { UTILS_PATH_ERROR("Failed to create fragment file", dir); return TILEDB_UT_ERR; } // Success return TILEDB_UT_OK; } int delete_dir(StorageFS *fs, const std::string& dirname) { if (fs->delete_dir(dirname)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int move_path(StorageFS *fs, const std::string& old_path, const std::string& new_path) { if (fs->move_path(old_path, new_path)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } template bool empty_value(T value) { if(&typeid(T) == &typeid(int)) return value == T(TILEDB_EMPTY_INT32); else if(&typeid(T) == &typeid(int64_t)) return value == T(TILEDB_EMPTY_INT64); else if(&typeid(T) == &typeid(float)) return value == T(TILEDB_EMPTY_FLOAT32); else if(&typeid(T) == &typeid(double)) return value == T(TILEDB_EMPTY_FLOAT64); else return false; } int expand_buffer(void*& buffer, size_t& buffer_allocated_size) { buffer_allocated_size *= 2; buffer = realloc(buffer, buffer_allocated_size); if(buffer == NULL) { UTILS_SYSTEM_ERROR("Cannot reallocate buffer"); return TILEDB_UT_ERR; } else { return TILEDB_UT_OK; } } template void expand_mbr(T* mbr, const T* coords, int dim_num) { for(int i=0; i coords[i]) mbr[2*i] = coords[i]; // Update upper bound on dimension i if(mbr[2*i+1] < coords[i]) mbr[2*i+1] = coords[i]; } } ssize_t file_size(StorageFS *fs, const std::string& filename) { return fs->file_size(filename); } std::string current_dir(StorageFS *fs) { return fs->current_dir(); } int set_working_dir(StorageFS *fs, const std::string& dir) { if (fs->is_dir(dir)) { return fs->set_working_dir(dir); } else { UTILS_ERROR("Failed to set_working_dir as "+dir+" does not exist"); return TILEDB_UT_ERR; } } std::vector get_dirs(StorageFS *fs, const std::string& dir) { return fs->get_dirs(dir); } std::vector get_files(StorageFS *fs, const std::string& dir) { return fs->get_files(dir); } std::vector get_fragment_dirs(StorageFS *fs, const std::string& dir) { std::vector dirs = get_dirs(fs, dir); std::vector fragment_dirs; for (auto const& dir: dirs) { if (is_fragment(fs, dir)) { fragment_dirs.push_back(dir); } } return fragment_dirs; } void gzip_handle_error(int rc, const std::string& message) { // Just listing all Z errors here for completion sake. // Note that deflate() and inflate() should not throw errors technically // but deflateInit() and inflateInit() can. switch (rc) { case Z_ERRNO: UTILS_SYSTEM_ERROR(message+": Z_ERRNO"); break; case Z_STREAM_ERROR: UTILS_ERROR(message+": Z_STREAM_ERROR"); break; case Z_DATA_ERROR: UTILS_ERROR(message+": Z_DATA_ERROR"); break; case Z_MEM_ERROR: UTILS_ERROR(message+": Z_MEM_ERROR"); break; case Z_BUF_ERROR: UTILS_ERROR(message+": Z_BUF_ERROR"); break; case Z_VERSION_ERROR: UTILS_ERROR(message+": Z_VERSION_ERROR"); break; default: UTILS_ERROR(message+": " + std::to_string(rc)); } } ssize_t gzip( unsigned char* in, size_t in_size, unsigned char* out, size_t out_size, const int level) { ssize_t ret; z_stream strm; // Allocate deflate state strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; ret = deflateInit(&strm, level); if(ret != Z_OK) { gzip_handle_error(ret, "Cannot compress with GZIP: deflateInit error"); (void)deflateEnd(&strm); return TILEDB_UT_ERR; } // Compress strm.next_in = in; strm.next_out = out; strm.avail_in = in_size; strm.avail_out = out_size; ret = deflate(&strm, Z_FINISH); // Clean up (void)deflateEnd(&strm); // Return if(ret == Z_STREAM_ERROR) { UTILS_ERROR("Encountered Z_STREAM_ERROR; Could not compress buffer; deflate error"); return TILEDB_UT_ERR; } else if (strm.avail_in != 0){ UTILS_ERROR("All input could not be compressed: deflate error"); return TILEDB_UT_ERR; } else { // Return size of compressed data return out_size - strm.avail_out; } } int gunzip( unsigned char* in, size_t in_size, unsigned char* out, size_t avail_out, size_t& out_size) { int ret; z_stream strm; // Allocate deflate state strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; ret = inflateInit(&strm); if(ret != Z_OK) { gzip_handle_error(ret, "Cannot decompress with GZIP: inflateInit error"); return TILEDB_UT_ERR; } // Decompress strm.next_in = in; strm.next_out = out; strm.avail_in = in_size; strm.avail_out = avail_out; ret = inflate(&strm, Z_FINISH); if(ret != Z_STREAM_END) { gzip_handle_error(ret, "Cannot decompress with GZIP: inflate error"); return TILEDB_UT_ERR; } // Clean up (void)inflateEnd(&strm); // Calculate size of compressed data out_size = avail_out - strm.avail_out; // Success return TILEDB_UT_OK; } template bool has_duplicates(const std::vector& v) { std::set s(v.begin(), v.end()); return s.size() != v.size(); } template bool inside_subarray(const T* coords, const T* subarray, int dim_num) { for(int i=0; i subarray[2*i+1]) return false; return true; } template bool intersect(const std::vector& v1, const std::vector& v2) { std::set s1(v1.begin(), v1.end()); std::set s2(v2.begin(), v2.end()); std::vector intersect; std::set_intersection(s1.begin(), s1.end(), s2.begin(), s2.end(), std::back_inserter(intersect)); return intersect.size() != 0; } bool is_array(StorageFS *fs, const std::string& dir) { // Check existence return fs->is_file(fs->append_paths(dir, TILEDB_ARRAY_SCHEMA_FILENAME)); } template bool is_contained( const T* range_A, const T* range_B, int dim_num) { for(int i=0; i range_B[2*i+1]) return false; return true; } bool is_dir(StorageFS *fs, const std::string& dir) { return fs->is_dir(dir); } bool is_file(StorageFS *fs, const std::string& file) { return fs->is_file(file); } bool is_fragment(StorageFS *fs, const std::string& dir) { // Check existence return fs->is_file(fs->append_paths(dir, TILEDB_FRAGMENT_FILENAME)); } bool is_group(StorageFS *fs, const std::string& dir) { // Check existence return fs->is_file(fs->append_paths(dir, TILEDB_GROUP_FILENAME)); } bool is_metadata(StorageFS *fs, const std::string& dir) { // Check existence return fs->is_file(fs->append_paths(dir, TILEDB_METADATA_SCHEMA_FILENAME)); } bool is_positive_integer(const char* s) { int i=0; if(s[0] == '-') // negative return false; if(s[0] == '0' && s[1] == '\0') // equal to 0 return false; if(s[0] == '+') i = 1; // Skip the first character if it is the + sign for(; s[i] != '\0'; ++i) { if(!isdigit(s[i])) return false; } return true; } template bool is_unary_subarray(const T* subarray, int dim_num) { for(int i=0; iis_file(fs->append_paths(dir, TILEDB_WORKSPACE_FILENAME)); } #ifdef HAVE_MPI int mpi_io_read_from_file( const MPI_Comm* mpi_comm, const std::string& filename, off_t offset, void* buffer, size_t length) { // Sanity check if(mpi_comm == NULL) { UTILS_ERROR("Invalid MPI communicator"); return TILEDB_UT_ERR; } // Open file MPI_File fh; if(MPI_File_open( *mpi_comm, (char*) filename.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &fh)) { UTILS_PATH_ERROR("Cannot read from file; MPI file opening error", filename); return TILEDB_UT_ERR; } // Read MPI_File_seek(fh, offset, MPI_SEEK_SET); MPI_Status mpi_status; if(MPI_File_read(fh, buffer, length, MPI_CHAR, &mpi_status)) { UTILS_PATH_ERROR("Cannot read from file; MPI file reading error", filename); return TILEDB_UT_ERR; } // Close file if(MPI_File_close(&fh)) { UTILS_PATH_ERROR("Cannot read from file; MPI file closing error", filename); return TILEDB_UT_ERR; } // Success return TILEDB_UT_OK; } int mpi_io_write_to_file( const MPI_Comm* mpi_comm, const char* filename, const void* buffer, size_t buffer_size) { // Open file MPI_File fh; if(MPI_File_open( *mpi_comm, (char*) filename, MPI_MODE_WRONLY | MPI_MODE_APPEND | MPI_MODE_CREATE | MPI_MODE_SEQUENTIAL, MPI_INFO_NULL, &fh)) { UTILS_PATH_ERROR("Cannot write to file; MPI file opening error", filename); return TILEDB_UT_ERR; } // Append attribute data to the file in batches of // TILEDB_UT_MAX_WRITE_COUNT bytes at a time MPI_Status mpi_status; while(buffer_size > TILEDB_UT_MAX_WRITE_COUNT) { if(MPI_File_write( fh, (void*) buffer, TILEDB_UT_MAX_WRITE_COUNT, MPI_CHAR, &mpi_status)) { UTILS_PATH_ERROR("Cannot write to file; MPI file writing error", filename); return TILEDB_UT_ERR; } buffer_size -= TILEDB_UT_MAX_WRITE_COUNT; } if(MPI_File_write(fh, (void*) buffer, buffer_size, MPI_CHAR, &mpi_status)) { UTILS_PATH_ERROR("Cannot write to file; MPI file writing error", filename); return TILEDB_UT_ERR; } // Close file if(MPI_File_close(&fh)) { UTILS_PATH_ERROR("Cannot write to file; MPI file closing error", filename); return TILEDB_UT_ERR; } // Success return TILEDB_UT_OK; } int mpi_io_sync( StorageFS *fs, const MPI_Comm* mpi_comm, const char* filename) { // Open file MPI_File fh; int rc; if(is_dir(fs, filename)) // DIRECTORY rc = MPI_File_open( *mpi_comm, (char*) filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh); else if(is_file(fs, filename)) // FILE rc = MPI_File_open( *mpi_comm, (char*) filename, MPI_MODE_WRONLY | MPI_MODE_APPEND | MPI_MODE_CREATE | MPI_MODE_SEQUENTIAL, MPI_INFO_NULL, &fh); else return TILEDB_UT_OK; // If file does not exist, exit // Handle error if(rc) { UTILS_PATH_ERROR("Cannot sync file; MPI file opening error", filename); return TILEDB_UT_ERR; } // Sync if(MPI_File_sync(fh)) { UTILS_PATH_ERROR("Cannot sync file; MPI file syncing error", filename); return TILEDB_UT_ERR; } // Close file if(MPI_File_close(&fh)) { UTILS_PATH_ERROR("Cannot sync file; MPI file closing error", filename); return TILEDB_UT_ERR; } // Success return TILEDB_UT_OK; } #endif #ifdef HAVE_OPENMP int mutex_destroy(omp_lock_t* mtx) { omp_destroy_lock(mtx); return TILEDB_UT_OK; } int mutex_init(omp_lock_t* mtx) { omp_init_lock(mtx); return TILEDB_UT_OK; } int mutex_lock(omp_lock_t* mtx) { omp_set_lock(mtx); return TILEDB_UT_OK; } int mutex_unlock(omp_lock_t* mtx) { omp_unset_lock(mtx); return TILEDB_UT_OK; } #endif int mutex_destroy(pthread_mutex_t* mtx) { reset_errno(); if(pthread_mutex_destroy(mtx) != 0) { UTILS_SYSTEM_ERROR("Cannot destroy mutex"); return TILEDB_UT_ERR; } else { return TILEDB_UT_OK; } } int mutex_init(pthread_mutex_t* mtx) { reset_errno(); pthread_mutexattr_t mtx_attr; if (pthread_mutexattr_init(&mtx_attr)) { UTILS_SYSTEM_ERROR("Cannot initialize mutex attribute"); return TILEDB_UT_ERR; } if (pthread_mutexattr_settype(&mtx_attr, PTHREAD_MUTEX_ERRORCHECK)) { pthread_mutexattr_destroy(&mtx_attr); UTILS_SYSTEM_ERROR("Cannot set mutex attribute type"); return TILEDB_UT_ERR; } if (pthread_mutex_init(mtx, NULL) != 0) { pthread_mutexattr_destroy(&mtx_attr); UTILS_SYSTEM_ERROR("Cannot initialize mutex"); return TILEDB_UT_ERR; } if (pthread_mutexattr_destroy(&mtx_attr)) { UTILS_SYSTEM_ERROR("Cannot destroy mutex attribute"); return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int mutex_lock(pthread_mutex_t* mtx) { reset_errno(); if(pthread_mutex_lock(mtx) != 0) { UTILS_SYSTEM_ERROR("Cannot lock mutex"); return TILEDB_UT_ERR; } else { return TILEDB_UT_OK; } } int mutex_unlock(pthread_mutex_t* mtx) { reset_errno(); if(pthread_mutex_unlock(mtx) != 0) { UTILS_SYSTEM_ERROR("Cannot unlock mutex"); return TILEDB_UT_ERR; } else { return TILEDB_UT_OK; } } std::string parent_dir(StorageFS *fs, const std::string& dir) { // Get real dir std::string real_dir; if (fs == NULL) { // Allow fs to be NULL for support to parent_dir in tiledb_storage.h real_dir = dir; } else { real_dir = fs->real_dir(dir); } // Start from the end of the string int pos = real_dir.size() - 1; // Skip the potential last '/' if(real_dir[pos] == '/') --pos; std::size_t query_index = real_dir.find("?"); pos = query_index == std::string::npos ? pos : query_index; // Scan backwords until you find the next '/' while(pos > 0 && real_dir[pos] != '/') --pos; if(query_index == std::string::npos) return real_dir.substr(0, pos); else return (real_dir.substr(0,pos)).append("/" + real_dir.substr(query_index)); } int read_from_file(StorageFS *fs, const std::string& filename, off_t offset, void* buffer, size_t length) { if (fs->read_from_file(filename, offset, buffer, length)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } std::string real_dir(StorageFS *fs, const std::string& dir) { return fs->real_dir(dir); } int64_t RLE_compress( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size) { // Initializations int cur_run_len = 1; int max_run_len = 65535; const unsigned char* input_cur = input + value_size; const unsigned char* input_prev = input; unsigned char* output_cur = output; int64_t value_num = input_size / value_size; int64_t output_size = 0; size_t run_size = value_size + 2*sizeof(char); unsigned char byte; // Trivial case if(value_num == 0) return 0; // Sanity check on input buffer if(input_size % value_size) { UTILS_ERROR("Failed compressing with RLE; invalid input buffer format"); return TILEDB_UT_ERR; } // Make runs for(int64_t i=1; i output_allocated_size) { UTILS_ERROR("Failed compressing with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // Copy to output buffer memcpy(output_cur, input_prev, value_size); output_cur += value_size; byte = (unsigned char) (cur_run_len >> 8); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); byte = (unsigned char) (cur_run_len % 256); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); output_size += run_size; // Reset current run length cur_run_len = 1; } // Update run info input_prev = input_cur; input_cur = input_prev + value_size; } // Save final run // --- Sanity check on size if(output_size + run_size > output_allocated_size) { UTILS_ERROR("Failed compressing with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // --- Copy to output buffer memcpy(output_cur, input_prev, value_size); output_cur += value_size; byte = (unsigned char) (cur_run_len >> 8); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); byte = (unsigned char) (cur_run_len % 256); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); output_size += run_size; // Success return output_size; } size_t RLE_compress_bound(size_t input_size, size_t value_size) { // In the worst case, RLE adds two bytes per every value in the buffer. int64_t value_num = input_size / value_size; return input_size + value_num * 2; } size_t RLE_compress_bound_coords( size_t input_size, size_t value_size, int dim_num) { // In the worst case, RLE adds two bytes per every value in the buffer for // each of its dim_num-1 coordinates (one dimension is never compressed). // The last sizeof(int64_t) is to record the number of cells compressed. int64_t cell_num = input_size / (dim_num*value_size); return input_size + cell_num * (dim_num-1) * 2 + sizeof(int64_t); } int64_t RLE_compress_coords_col( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size, int dim_num) { // Initializations int cur_run_len; int max_run_len = 65535; const unsigned char* input_cur; const unsigned char* input_prev = input; unsigned char* output_cur = output; size_t coords_size = value_size*dim_num; size_t run_size = value_size + 2*sizeof(char); int64_t coords_num = input_size / coords_size; int64_t output_size = 0; unsigned char byte; // Sanity check on input buffer format if(input_size % coords_size) { UTILS_ERROR("Failed compressing coordinates with RLE; invalid buffer format"); return TILEDB_UT_ERR; } // Trivial case if(coords_num == 0) return 0; // Copy the number of coordinates if(output_size + sizeof(int64_t) > output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } memcpy(output_cur, &coords_num, sizeof(int64_t)); output_cur += sizeof(int64_t); output_size += sizeof(int64_t); // Copy the first dimension intact // --- Sanity check on size if(output_size + coords_num*value_size > output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // --- Copy to output buffer for(int64_t i=0; i output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // Copy to output buffer memcpy(output_cur, input_prev, value_size); output_cur += value_size; byte = (unsigned char) (cur_run_len >> 8); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); byte = (unsigned char) (cur_run_len % 256); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); output_size += value_size + 2*sizeof(char); // Update current run length cur_run_len = 1; } // Update run info input_prev = input_cur; input_cur = input_prev + coords_size; } // Save final run //--- Sanity check on ouput size if(output_size + run_size > output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // --- Copy to output buffer memcpy(output_cur, input_prev, value_size); output_cur += value_size; byte = (unsigned char) (cur_run_len >> 8); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); byte = (unsigned char) (cur_run_len % 256); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); output_size += value_size + 2*sizeof(char); } // Success return output_size; } int64_t RLE_compress_coords_row( const unsigned char* input, size_t input_size, unsigned char* output, size_t output_allocated_size, size_t value_size, int dim_num) { // Initializations int cur_run_len; int max_run_len = 65535; const unsigned char* input_cur; const unsigned char* input_prev; unsigned char* output_cur = output; size_t coords_size = value_size*dim_num; int64_t coords_num = input_size / coords_size; int64_t output_size = 0; size_t run_size = value_size + 2*sizeof(char); unsigned char byte; // Sanity check on input buffer format if(input_size % coords_size) { UTILS_ERROR("Failed compressing coordinates with RLE; invalid buffer format"); return TILEDB_UT_ERR; } // Trivial case if(coords_num == 0) return 0; // Copy the number of coordinates if(output_size + sizeof(int64_t) > output_allocated_size) { UTILS_ERROR("Filed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } memcpy(output_cur, &coords_num, sizeof(int64_t)); output_cur += sizeof(int64_t); output_size += sizeof(int64_t); // Make runs for each of the first (dim_num-1) dimensions for(int d=0; d output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // Copy to output buffer memcpy(output_cur, input_prev, value_size); output_cur += value_size; byte = (unsigned char) (cur_run_len >> 8); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); byte = (unsigned char) (cur_run_len % 256); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); output_size += value_size + 2*sizeof(char); // Update current run length cur_run_len = 1; } // Update run info input_prev = input_cur; input_cur = input_prev + coords_size; } // Save final run // --- Sanity check on size if(output_size + run_size > output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // --- Copy to output buffer memcpy(output_cur, input_prev, value_size); output_cur += value_size; byte = (unsigned char) (cur_run_len >> 8); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); byte = (unsigned char) (cur_run_len % 256); memcpy(output_cur, &byte, sizeof(char)); output_cur += sizeof(char); output_size += run_size; } // Copy the final dimension intact // --- Sanity check on size if(output_size + coords_num*value_size > output_allocated_size) { UTILS_ERROR("Failed compressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // --- Copy to output buffer input_prev = input + (dim_num-1)*value_size; for(int64_t i=0; i output_allocated_size) { UTILS_ERROR("Failed decompressing with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // Copy to output buffer for(int64_t j=0; j input_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; input buffer overflow"); return TILEDB_UT_ERR; } // --- Copy number of coordinates memcpy(&coords_num, input_cur, sizeof(int64_t)); input_cur += sizeof(int64_t); input_offset += sizeof(int64_t); // Trivial case if(coords_num == 0) return TILEDB_UT_OK; // Copy the first dimension intact // --- Sanity check on output buffer size if(coords_num * coords_size > output_allocated_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // --- Sanity check on output buffer size if(input_offset + coords_num * value_size > input_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; input buffer overflow"); return TILEDB_UT_ERR; } // --- Copy first dimension to output for(int64_t i=0; i input_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; input buffer overflow"); return TILEDB_UT_ERR; } // --- Copy number of coordinates memcpy(&coords_num, input_cur, sizeof(int64_t)); input_cur += sizeof(int64_t); input_offset += sizeof(int64_t); // Trivial case if(coords_num == 0) return TILEDB_UT_OK; // Sanity check on output buffer size if(coords_num * coords_size > output_allocated_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; output buffer overflow"); return TILEDB_UT_ERR; } // Get number of runs int64_t run_num = (input_size - input_offset - coords_num * value_size) / run_size; // Sanity check on input buffer format if((input_size - input_offset - coords_num * value_size) % run_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; invalid input buffer format"); return TILEDB_UT_ERR; } // Decompress runs for each of the first (dim_num-1) dimensions int64_t coords_i = 0; int d = 0; for(int64_t i=0; i input_size) { UTILS_ERROR("Failed decompressing coordinates with RLE; input buffer overflow"); return TILEDB_UT_ERR; } // --- Copy to output buffer for(int64_t i=0; i value.size()) return false; return std::equal(prefix.begin(), prefix.end(), value.begin()); } int sync_path(StorageFS *fs, const std::string& path) { if (fs->sync_path(path)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int close_file(StorageFS *fs, const std::string& filename) { if (fs->close_file(filename)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int write_to_file( StorageFS *fs, const std::string& filename, const void* buffer, size_t buffer_size) { if (fs->write_to_file(filename, buffer, buffer_size)) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } return TILEDB_UT_OK; } int delete_directories(StorageFS *fs, const std::vector& directories) { // Delete old fragments for(auto i=0u; idelete_dir(directories[i]) != TILEDB_UT_OK) { tiledb_ut_errmsg = tiledb_fs_errmsg; return TILEDB_UT_ERR; } } return TILEDB_UT_OK; } // Explicit template instantiations template int64_t cell_num_in_subarray( const int* subarray, int dim_num); template int64_t cell_num_in_subarray( const int64_t* subarray, int dim_num); template int64_t cell_num_in_subarray( const float* subarray, int dim_num); template int64_t cell_num_in_subarray( const double* subarray, int dim_num); template bool cell_in_subarray( const int* cell, const int* subarray, int dim_num); template bool cell_in_subarray( const int64_t* cell, const int64_t* subarray, int dim_num); template bool cell_in_subarray( const float* cell, const float* subarray, int dim_num); template bool cell_in_subarray( const double* cell, const double* subarray, int dim_num); template int cmp_col_order( const int* coords_a, const int* coords_b, int dim_num); template int cmp_col_order( const int64_t* coords_a, const int64_t* coords_b, int dim_num); template int cmp_col_order( const float* coords_a, const float* coords_b, int dim_num); template int cmp_col_order( const double* coords_a, const double* coords_b, int dim_num); template int cmp_col_order( int64_t id_a, const int* coords_a, int64_t id_b, const int* coords_b, int dim_num); template int cmp_col_order( int64_t id_a, const int64_t* coords_a, int64_t id_b, const int64_t* coords_b, int dim_num); template int cmp_col_order( int64_t id_a, const float* coords_a, int64_t id_b, const float* coords_b, int dim_num); template int cmp_col_order( int64_t id_a, const double* coords_a, int64_t id_b, const double* coords_b, int dim_num); template int cmp_row_order( const int* coords_a, const int* coords_b, int dim_num); template int cmp_row_order( const int64_t* coords_a, const int64_t* coords_b, int dim_num); template int cmp_row_order( const float* coords_a, const float* coords_b, int dim_num); template int cmp_row_order( const double* coords_a, const double* coords_b, int dim_num); template int cmp_row_order( int64_t id_a, const int* coords_a, int64_t id_b, const int* coords_b, int dim_num); template int cmp_row_order( int64_t id_a, const int64_t* coords_a, int64_t id_b, const int64_t* coords_b, int dim_num); template int cmp_row_order( int64_t id_a, const float* coords_a, int64_t id_b, const float* coords_b, int dim_num); template int cmp_row_order( int64_t id_a, const double* coords_a, int64_t id_b, const double* coords_b, int dim_num); template bool empty_value(int value); template bool empty_value(int64_t value); template bool empty_value(float value); template bool empty_value(double value); template void expand_mbr( int* mbr, const int* coords, int dim_num); template void expand_mbr( int64_t* mbr, const int64_t* coords, int dim_num); template void expand_mbr( float* mbr, const float* coords, int dim_num); template void expand_mbr( double* mbr, const double* coords, int dim_num); template bool has_duplicates(const std::vector& v); template bool inside_subarray( const int* coords, const int* subarray, int dim_num); template bool inside_subarray( const int64_t* coords, const int64_t* subarray, int dim_num); template bool inside_subarray( const float* coords, const float* subarray, int dim_num); template bool inside_subarray( const double* coords, const double* subarray, int dim_num); template bool intersect( const std::vector& v1, const std::vector& v2); template bool is_contained( const int* range_A, const int* range_B, int dim_num); template bool is_contained( const int64_t* range_A, const int64_t* range_B, int dim_num); template bool is_contained( const float* range_A, const float* range_B, int dim_num); template bool is_contained( const double* range_A, const double* range_B, int dim_num); template bool is_unary_subarray(const int* subarray, int dim_num); template bool is_unary_subarray(const int64_t* subarray, int dim_num); template bool is_unary_subarray(const float* subarray, int dim_num); template bool is_unary_subarray(const double* subarray, int dim_num); genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/000077500000000000000000000000001453617025200215305ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_azure_blob.cc000066400000000000000000000540751453617025200257220ustar00rootroot00000000000000/** * @file storage_azure_blob.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2019-2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Azure Blob Support for StorageFS */ #include "storage_azure_blob.h" #include "error.h" #include "uri.h" #include "utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __APPLE__ #if !defined(ELIBACC) #define ELIBACC -1 #endif #endif #define AZ_BLOB_ERROR(MSG, PATH) SYSTEM_ERROR(TILEDB_FS_ERRMSG, "Azure: "+MSG, PATH, tiledb_fs_errmsg) using namespace azure::storage_lite; static std::string run_command(const std::string& command) { std::string output; std::array buffer; std::unique_ptr pipe(popen(command.data(), "r"), pclose); if (pipe) { while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { output += buffer.data(); } } return output; } static std::string get_account_key(const std::string& account_name) { // Try environment variables AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY first char *az_storage_ac_name = getenv("AZURE_STORAGE_ACCOUNT"); if (!az_storage_ac_name || (az_storage_ac_name && account_name.compare(az_storage_ac_name) == 0)) { char *key = getenv("AZURE_STORAGE_KEY"); if (key) { return key; } } // Try retrieving first account key via az CLI std::string account_key; std::string key1 = run_command("az storage account keys list --query \"[?keyName == 'key1'].value | [0]\" -o tsv --account-name " + account_name); if (key1.length() > 1) { // Remove newlines std::regex pattern("(.*)\\r?\\n?"); std::smatch match; if (std::regex_match(key1, match, pattern) && match.ready() && !match.empty() && match.size() == 2) { try { auto matched = match[1].str(); // Check if it is really an encoded key azure::storage_lite::from_base64(matched); account_key = matched; } catch(...) { // Ignore } } } return account_key; } static std::string get_sas_token(const std::string& account_name) { // Try environment variables AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY first char *az_storage_ac_name = getenv("AZURE_STORAGE_ACCOUNT"); if (!az_storage_ac_name || (az_storage_ac_name && account_name.compare(az_storage_ac_name) == 0)) { char *key = getenv("AZURE_STORAGE_SAS_TOKEN"); if (key) { return key; } } return ""; } static std::string get_blob_endpoint(const std::string& endpoint, const std::string& account) { // Get enviroment variable for AZURE_STORAGE_SERVICE_ENDPOINT std::string az_blob_endpoint(endpoint); if (az_blob_endpoint.empty() || az_blob_endpoint.compare(account + ".blob") == 0) { char* az_blob_endpoint_env = getenv("AZURE_STORAGE_SERVICE_ENDPOINT"); if (az_blob_endpoint_env) { az_blob_endpoint = az_blob_endpoint_env; } else { az_blob_endpoint = ""; } } return az_blob_endpoint; } static std::string get_access_token(const std::string& account_name, const std::string& path) { // Invoke `az account get-access-token --resource // https://.blob.core.windows.net -o tsv --query accessToken` Can // probably use `az account get-access-token --resource // https://storage.azure.com/ -o tsv --query accessToken` too std::string endpoint = path; std::size_t scheme_pos = endpoint.find("://"); std::string resource_url = "https://"; if (scheme_pos != std::string::npos) { resource_url.append(endpoint.substr(scheme_pos + 3)); } else { resource_url.append(account_name + ".blob.core.windows.net"); } std::string command = "az account get-access-token --resource " + resource_url + " -o tsv --query accessToken"; return run_command(command); } std::string AzureBlob::get_path(const std::string& path) { std::string pathname(path); if (path.find("://") != std::string::npos) { azure_uri uri_path(path); pathname = uri_path.path(); // This is the container if (pathname.empty()) { return ""; } } if (pathname[0] == '/') { return pathname.substr(1); } if (pathname.empty()) { return working_dir_; } else if (starts_with(pathname, working_dir_)) { // TODO: this is a hack for now, but should work fine with GenomicsDB. return pathname; } else { return working_dir_ + '/' + pathname; } } AzureBlob::AzureBlob(const std::string& home) { azure_uri path_uri(home); // az://@.blob.core.windows.net/ // e.g. az://test@mytest.blob.core.windows.net/ws // azb://?> // e.g. azb://test/ws?account=mytest&endpoint=mytest.blob.core.windows.net if (path_uri.protocol().compare("az") != 0 && path_uri.protocol().compare("azb") != 0) { throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "Azure Blob FS only supports az:// or azb:// URI protocols"); } std::string azure_account = path_uri.account(); if (azure_account.empty() && path_uri.protocol().compare("azb") == 0) { char* az_storage_account_env = getenv("AZURE_STORAGE_ACCOUNT"); if (az_storage_account_env) azure_account = az_storage_account_env; } if (azure_account.size() == 0 || path_uri.container().size() == 0) { throw std::system_error(EPROTO, std::generic_category(), "Azure Blob URI does not seem to have either an account or a container"); } // Algorithm to get azure storage credentials. Try AZURE_STORAGE_ACCOUNT_KEY first, followed by AZURE_STORAGE_SAS_TOKEN and last // try getting an access token directly from CLI std::shared_ptr cred = nullptr; std::string azure_account_key = get_account_key(azure_account); if (!azure_account_key.empty()) { cred = std::make_shared(azure_account, azure_account_key); } else { std::string azure_sas_token = get_sas_token(azure_account); if (!azure_sas_token.empty()) { cred = std::make_shared(azure_sas_token); } else { // Last ditch effort, try getting an access token directly from CLI. AZ_BLOB_ERROR("Could not authenticate via AZURE_STORAGE_KEY or AZURE_STORAGE_SAS_TOKEN env vars. " + "Trying to get access token directly via CLI", home); std::string token = get_access_token(azure_account, home); if (token.size() > 0) { cred = std::make_shared(token); } } } if (cred == nullptr) { throw std::system_error(EIO, std::generic_category(), "Could not get credentials for azure storage account=" + azure_account + ". " + "Try setting environment variables AZURE_STORAGE_KEY or AZURE_STORAGE_SAS_TOKEN before restarting operation"); } std::shared_ptr account = std::make_shared( azure_account, cred, /* use_https */ true, get_blob_endpoint(path_uri.endpoint(), azure_account)); auto num_threads = getenv("TILEDB_NUM_THREADS"); if (num_threads) { num_threads_ = std::string(num_threads)=="0"?1:std::stoi(num_threads); if (!num_threads_) num_threads_ = 1; } #ifdef DEBUG std::cerr << "*** Using threads=" << num_threads_ << " with azure SDK client" << std::endl; #endif std::string ca_certs_location = locate_ca_certs(); if (ca_certs_location.empty()) { blob_client_ = std::make_shared(account, 1/*concurrency*/); } else { blob_client_ = std::make_shared(account, 1/*concurrency*/, ca_certs_location); } bc_wrapper_ = std::make_shared(blob_client_); blob_client_wrapper_ = reinterpret_cast(bc_wrapper_.get()); /* if (!blob_client_wrapper_->container_exists(path_uri.container())) { AZ_BLOB_ERROR("Container does not seem to exist", path_uri.container()); throw std::system_error(EIO, std::generic_category(), "AzureBlobFS only supports accessible and already existing containers"); } */ account_name_ = azure_account; container_name_ = path_uri.container(); working_dir_ = get_path(path_uri.path()); adls_client_ = std::make_shared(account, num_threads_, false); // Set default buffer sizes, overridden with env vars TILEDB_DOWNLOAD_BUFFER_SIZE and TILEDB_UPLOAD_BUFFER_SIZE download_buffer_size_ = constants::default_block_size; // 8M upload_buffer_size_ = constants::default_block_size; // 8M auto max_stream_size_var = getenv("TILEDB_MAX_STREAM_SIZE"); if (max_stream_size_var) { max_stream_size_ = std::stoll(max_stream_size_var); } } std::string AzureBlob::current_dir() { return working_dir_; } int AzureBlob::set_working_dir(const std::string& dir) { working_dir_ = get_path(dir); return TILEDB_FS_OK; } bool AzureBlob::path_exists(const std::string& path) { auto blob_property = blob_client_wrapper_->get_blob_property(container_name_, get_path(path)); if (blob_property.valid()) { if (blob_property.content_type.empty() && path.back() == '/') { return true; } else if (!blob_property.content_type.empty() && path.back() != '/') { return true; } } else if (path.back() == '/') { // Check directories in non-hierarchical namespaces by checking for children as they are not explicitly // created as in hierarchical namespaces auto response = blob_client_wrapper_->list_blobs_segmented(container_name_, "/", "", get_path(path), 1); return response.blobs.size() > 0; } return false; } std::string AzureBlob::real_dir(const std::string& dir) { if (dir.find("://") != std::string::npos) { azure_uri path_uri(dir); std::string account = path_uri.account(); if (account.empty() && path_uri.protocol().compare("azb") == 0) { char* az_storage_account_env = getenv("AZURE_STORAGE_ACCOUNT"); if (az_storage_account_env) account = az_storage_account_env; } if (account.compare(account_name_) || path_uri.container().compare(container_name_)) { throw std::runtime_error( "Credentialed account during instantiation does not match the uri " "passed to real_dir. Aborting"); } } return get_path(dir); } int AzureBlob::create_path(const std::string& path) { return write_to_file(path, NULL, 0); } int AzureBlob::create_dir(const std::string& dir) { if (is_file(dir)) { AZ_BLOB_ERROR("Path already exists", dir); return TILEDB_FS_ERR; } // No support in the azure lite client to create a folder // no-op for this case for now return TILEDB_FS_OK; } int AzureBlob::delete_dir(const std::string& dir) { int rc = TILEDB_FS_OK; adls_client_->delete_directory(container_name_, get_path(dir)); if (errno > 0) { // Try again using the blob client directly for non-hierarchical filesystems std::string continuation_token = ""; auto bclient = reinterpret_cast(blob_client_.get()); auto response = blob_client_wrapper_->list_blobs_segmented(container_name_, "/", continuation_token, slashify(get_path(dir)), INT_MAX); do { for (auto i=0u; idelete_blob(container_name_, response.blobs[i].name, false).get(); if (!result.success()) { AZ_BLOB_ERROR(result.error().message, response.blobs[i].name); rc = TILEDB_FS_ERR; } } } } while (!continuation_token.empty()); } return rc; } std::vector AzureBlob::get_dirs(const std::string& dir) { std::vector dirs; std::string continuation_token = ""; auto response = blob_client_wrapper_->list_blobs_segmented(container_name_, "/", continuation_token, slashify(get_path(dir)), INT_MAX); do { for (auto i=0u; i AzureBlob::get_files(const std::string& dir) { std::vector files; std::string continuation_token = ""; auto response = blob_client_wrapper_->list_blobs_segmented(container_name_, "/", continuation_token, slashify(get_path(dir)), INT_MAX); do { for (auto i=0u; idelete_blob(container_name_, get_path(filename)); return TILEDB_FS_OK; } ssize_t AzureBlob::file_size(const std::string& filename) { auto blob_property = blob_client_wrapper_->get_blob_property(container_name_, get_path(filename)); if (blob_property.valid() && !blob_property.content_type.empty()) { #if 0 if (filename.find_last_of(".json") != std::string::npos) { std::cerr << "Blob " << filename << " md5=" << blob_property.content_md5 << " size=" << blob_property.size<< std::endl; } #endif return blob_property.size; } else { #if 0 std::cerr << "No blob properties found for file=" << filename << std::endl; #endif return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int AzureBlob::read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length) { if (length == 0) { return TILEDB_FS_OK; // Nothing to read } std::string path = get_path(filename); auto bclient = reinterpret_cast(blob_client_.get()); storage_outcome read_result; // Heuristic: if the file can be contained in a block use download_blob_to_stream(), otherwise use the parallel download_blob_to_buffer() if (length <= max_stream_size_ || num_threads_ == 1) { omemstream os_buf(buffer, length); read_result = bclient->download_blob_to_stream(container_name_, path, offset, length, os_buf).get(); } else { try { read_result = bclient->download_blob_to_buffer(container_name_, path, offset, length, reinterpret_cast(buffer), num_threads_).get(); } catch (const std::exception& ex) { // Catch random exceptions from download_blob_to_buffer. Bug?? std::string message = "Random error from azure sdk with the download_blob_to_buffer api : " + std::string(ex.what()) + "\n current max_stream_size=" + std::to_string(max_stream_size_) + "bytes. " + "Try increasing the max_stream_size using the TILEDB_MAX_STREAM_SIZE environment variable in bytes"; AZ_BLOB_ERROR(message, filename); return TILEDB_FS_ERR; } } if (!read_result.success()) { AZ_BLOB_ERROR(read_result.error().message, filename); return TILEDB_FS_ERR; } else { return TILEDB_FS_OK; } } #define GRAIN_SIZE (4*1024*1024) // This method is based on upload_block_blob_from_buffer from the SDK except for the put_block_list stage which happens in commit_path() now std::future> AzureBlob::upload_block_blob(const std::string &blob, uint64_t block_size, int num_blocks, std::vector block_ids, const char* buffer, uint64_t bufferlen, int parallelism) { parallelism = std::min(num_blocks, parallelism); struct concurrent_task_info { std::string blob; const char* buffer; uint64_t blob_size; uint64_t block_size; int num_blocks; std::vector> metadata; }; struct concurrent_task_context { std::atomic num_workers{ 0 }; std::atomic block_index{ 0 }; std::atomic failed{ false }; storage_error failed_reason; std::promise> task_promise; std::vector> task_futures; }; auto info = std::make_shared(concurrent_task_info{ blob, buffer, bufferlen, block_size, num_blocks, empty_metadata }); auto context = std::make_shared(); context->num_workers = parallelism; auto thread_upload_func = [this, block_ids, info, context]() { while (true) { int i = context->block_index.fetch_add(1); if (i >= info->num_blocks || context->failed) { break; } const char* block_buffer = info->buffer + info->block_size * i; uint64_t block_size = std::min(info->block_size, info->blob_size - info->block_size * i); auto result = this->blob_client_.get()->upload_block_from_buffer(container_name_, info->blob, block_ids[i], block_buffer, block_size).get(); if (!result.success() && !context->failed.exchange(true)) { context->failed_reason = result.error(); } } if (context->num_workers.fetch_sub(1) == 1) { // I'm the last worker thread context->task_promise.set_value(context->failed ? storage_outcome(context->failed_reason) : storage_outcome()); } }; for (int i = 0; i < parallelism; ++i) { context->task_futures.emplace_back(std::async(std::launch::async, thread_upload_func)); } for (int i = 0; i < parallelism; ++i) { context->task_futures[i].get(); } return context->task_promise.get_future(); } int AzureBlob::write_to_file(const std::string& filename, const void *buffer, size_t buffer_size) { std::string path = get_path(filename); auto bclient = reinterpret_cast(blob_client_.get()); if (buffer_size == 0) { const std::lock_guard lock(write_map_mtx_); if (!blob_client_wrapper_->blob_exists(container_name_, path)) { auto result = bclient->create_append_blob(container_name_, path).get(); if (!result.success()) { AZ_BLOB_ERROR("Could not create zero length file: " + result.error().message, path); return TILEDB_FS_ERR; } else { return TILEDB_FS_OK; } } return TILEDB_FS_OK; } if (buffer_size > constants::max_num_blocks * constants::max_block_size) { AZ_BLOB_ERROR("Buffer size too large for azure upload", path); return TILEDB_FS_ERR; } #ifdef DEBUG update_expected_filesizes_map(path, buffer_size); #endif uint64_t block_size = buffer_size/constants::max_num_blocks; block_size = (block_size + GRAIN_SIZE-1)/GRAIN_SIZE*GRAIN_SIZE; block_size = std::min(block_size, constants::max_block_size); block_size = std::max(block_size, constants::default_block_size); int num_blocks = int((buffer_size + block_size-1)/block_size); auto block_ids = generate_block_ids(path, num_blocks); if (block_ids.size() == 0) { AZ_BLOB_ERROR("Could not get block_ids for upload_block_blob", path); return TILEDB_FS_ERR; } auto res = upload_block_blob(path, block_size, num_blocks, block_ids, reinterpret_cast(buffer), buffer_size, num_threads_).get(); if (!res.success()) { AZ_BLOB_ERROR(res.error().message, path); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int AzureBlob::commit_file(const std::string& path) { auto bclient = reinterpret_cast(blob_client_.get()); int rc = TILEDB_FS_OK; std::string filepath = get_path(path); const std::lock_guard lock(write_map_mtx_); auto search = write_map_.find(filepath); if (search != write_map_.end()) { std::vector> empty_metadata; auto put_result = bclient->put_block_list(container_name_, filepath, search->second, empty_metadata).get(); if (!put_result.success()) { AZ_BLOB_ERROR("Could not sync path with put_block_list: " + put_result.error().message, filepath); rc = TILEDB_FS_ERR; } write_map_.erase(search->first); #ifdef DEBUG if (!rc) { // Checks after commit, should we wait for propogation? if (!is_file(filepath)) { AZ_BLOB_ERROR("Could not find file after commit", path); rc = TILEDB_FS_ERR; } else { ssize_t expected = expected_filesize_from_map(filepath); ssize_t actual = file_size(path); if (expected != actual) { AZ_BLOB_ERROR("filesize after commit does not match with the expected filesize", path); std::string msg = std::string("expected=") + std::to_string(expected) + " actual=" + std::to_string(actual); AZ_BLOB_ERROR(msg, path); rc = TILEDB_FS_ERR; } } } filesizes_map_.erase(filepath); #endif } return rc; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_buffer.cc000066400000000000000000000335011453617025200250360ustar00rootroot00000000000000/** * @file storage_buffer.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020-2022 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the StorageBuffer class that buffers writes/reads to/from files */ #include "error.h" #include "storage_buffer.h" #include "utils.h" #include #include #include #include #define BUFFER_PATH_ERROR(MSG, PATH) free_buffer();SYSTEM_ERROR(TILEDB_BF_ERRMSG, MSG, PATH, tiledb_fs_errmsg) #define BUFFER_ERROR_WITH_ERRNO(MSG) free_buffer();TILEDB_ERROR_WITH_ERRNO(TILEDB_BF_ERRMSG, MSG, tiledb_fs_errmsg) #define BUFFER_ERROR(MSG) free_buffer();TILEDB_ERROR(TILEDB_BF_ERRMSG, MSG, tiledb_fs_errmsg) // Using a typical page size for chunks #define CHUNK 4096 StorageBuffer::StorageBuffer(StorageFS *fs, const std::string& filename, size_t chunk_size, const bool is_read) : fs_(fs), filename_(filename), read_only_(is_read) { if (read_only_) { if (fs_->file_size(filename) == TILEDB_FS_ERR) { BUFFER_PATH_ERROR("File does not seem to exist or is of zero length", filename_); is_error_ = true; } else { filesize_ = (size_t)fs_->file_size(filename); } } if (!(chunk_size_ = chunk_size)) { BUFFER_PATH_ERROR("Cannot perform buffered reads or writes as there is no buffer chunk size set", filename_); is_error_ = true; } } int StorageBuffer::read_buffer(void *bytes, size_t size) { // Nothing to do if (bytes == NULL || size == 0) { return TILEDB_BF_OK; } int rc = read_buffer(file_offset_, bytes, size); file_offset_ += size; return rc; } int StorageBuffer::read_buffer(off_t offset, void *bytes, size_t size) { // Nothing to do if (bytes == NULL || size == 0) { return TILEDB_BF_OK; } assert(read_only_); if (is_error_) { return TILEDB_BF_ERR; } if (offset + size > filesize_) { BUFFER_PATH_ERROR("Cannot read past the filesize from buffer", filename_); return TILEDB_BF_ERR; } if (buffer_ == NULL || !(offset>=buffer_offset_ && (offset+size)<=(buffer_offset_+buffer_size_))) { buffer_offset_ = (offset/CHUNK)*CHUNK; buffer_size_ = ((size/chunk_size_)+1)*chunk_size_ + (offset%CHUNK); // Factor in last chunk if (buffer_offset_+buffer_size_ > filesize_) { buffer_size_ = filesize_-buffer_offset_; } if (buffer_size_ > allocated_buffer_size_) { buffer_ = realloc(buffer_, buffer_size_); if (buffer_ == NULL) { BUFFER_ERROR_WITH_ERRNO("Cannot read to buffer; Mem allocation error"); return TILEDB_BF_ERR; } allocated_buffer_size_ = buffer_size_; } if (read_buffer()) { return TILEDB_BF_ERR; } } assert(offset >= buffer_offset_); assert(size <= buffer_size_); assert(size_t(offset-buffer_offset_) <= buffer_size_); void *pmem = memcpy(bytes, (char *)buffer_+offset-buffer_offset_, size); assert(pmem == bytes); return TILEDB_BF_OK; } int StorageBuffer::read_buffer() { if (fs_->read_from_file(filename_, buffer_offset_, (char *)buffer_, buffer_size_)) { BUFFER_PATH_ERROR("Cannot read to buffer", filename_); return TILEDB_BF_ERR; } return TILEDB_BF_OK; } int StorageBuffer::append_buffer(const void *bytes, size_t size) { assert(!read_only_); // Nothing to do if (bytes == NULL || size == 0) { return TILEDB_BF_OK; } if (is_error_) { return TILEDB_BF_ERR; } if (buffer_size_ >= chunk_size_) { assert(buffer_ != NULL); if (write_buffer()) { return TILEDB_BF_ERR; } } if (buffer_ == NULL || buffer_size_+size > allocated_buffer_size_) { size_t alloc_size = allocated_buffer_size_ + ((size/CHUNK)+1)*CHUNK; buffer_ = realloc(buffer_, alloc_size); if (buffer_ == NULL) { BUFFER_ERROR_WITH_ERRNO("Cannot write to buffer; Mem allocation error"); return TILEDB_BF_ERR; } allocated_buffer_size_ = alloc_size; } void *pmem = memcpy((char *)buffer_+buffer_size_, bytes, size); assert(pmem == (char *)buffer_+buffer_size_); buffer_size_ += size; return TILEDB_BF_OK; } int StorageBuffer::write_buffer() { if (is_error_) { return TILEDB_BF_ERR; } if (fs_->write_to_file(filename_, buffer_, buffer_size_)) { BUFFER_PATH_ERROR("Cannot write bytes", filename_); return TILEDB_BF_ERR; } buffer_size_ = 0; return TILEDB_BF_OK; } int StorageBuffer::finalize() { int rc = TILEDB_BF_OK; if (!read_only_) { rc = write_buffer(); } rc = fs_->close_file(filename_) || rc; if (rc) { // error is logged in write_buffer or close_file free_buffer(); return TILEDB_BF_ERR; } else { return TILEDB_BF_OK; } } int CompressedStorageBuffer::read_buffer(void *bytes, size_t size) { // Nothing to do if (bytes == NULL || size == 0) { return TILEDB_BF_OK; } assert(read_only_); if (is_error_) { return TILEDB_BF_ERR; } if (!buffer_) { switch (compression_type_) { case TILEDB_GZIP: if (gzip_read_buffer()) { BUFFER_PATH_ERROR("Cannot decompress and/or read bytes", filename_); return TILEDB_BF_ERR; } break; case TILEDB_NO_COMPRESSION: break; default: BUFFER_ERROR("Compression type=" + std::to_string(compression_type_) + " for read_buffer not supported for CompressedStorageBuffer"); return TILEDB_BF_ERR; } } int rc = StorageBuffer::read_buffer(file_offset_, bytes, size); file_offset_ += size; return rc; } int CompressedStorageBuffer::write_buffer() { if (is_error_) { return TILEDB_BF_ERR; } if (buffer_size_ > 0) { switch (compression_type_) { case TILEDB_GZIP: if (gzip_write_buffer()) { BUFFER_PATH_ERROR("Cannot compress and/or write bytes", filename_); return TILEDB_BF_ERR; } break; case TILEDB_NO_COMPRESSION: return StorageBuffer::write_buffer(); default: BUFFER_ERROR("Compression type=" + std::to_string(compression_type_) + " not supported in StorageBuffer"); return TILEDB_BF_ERR; } } buffer_size_ = 0; return TILEDB_BF_OK; } #define windowBits 15 #define GZIP_ENCODING 16 void gzip_handle_error(int rc, const std::string& message); int CompressedStorageBuffer::initialize_gzip_stream(z_stream *strm) { memset(strm, 0, sizeof(z_stream)); strm->zalloc = Z_NULL; strm->zfree = Z_NULL; strm->opaque = Z_NULL; strm->next_in = Z_NULL; int rc = TILEDB_BF_OK; if ((rc = inflateInit2(strm, (windowBits + 32))) != Z_OK) { gzip_handle_error(rc, std::string("Could not initialize decompression for file ")+filename_); free_buffer(); return TILEDB_BF_ERR; } return rc; } int CompressedStorageBuffer::gzip_read_buffer() { // gzip_read is done only once for a given file as we need all the bytes to decompress into buffer assert(!buffer_); /* allocate inflate state */ z_stream strm; if (initialize_gzip_stream(&strm)) return TILEDB_BF_ERR; unsigned char *in = (unsigned char *)malloc(chunk_size_); if (in == NULL) { BUFFER_ERROR_WITH_ERRNO( "Cannot read file into buffer; Mem allocation error for " + filename_ + " filesize=" + std::to_string(chunk_size_)); return TILEDB_BF_ERR; } unsigned char out[TILEDB_GZIP_CHUNK_SIZE]; int rc = TILEDB_BF_OK; size_t processed = 0; do { // decompress in chunks until input is exhausted memset(in, 0, chunk_size_); size_t read_size = filesize_-processed>chunk_size_?chunk_size_:filesize_-processed; if (fs_->read_from_file(filename_, processed, in, read_size) == TILEDB_UT_ERR) { free(in); BUFFER_PATH_ERROR("Could not read from file for decompression", filename_); return TILEDB_BF_ERR; } strm.avail_in = read_size; strm.next_in = in; // for next iteration, bump processed processed += read_size; // run inflate() on input until output buffer not full unsigned have; do { strm.avail_out = TILEDB_GZIP_CHUNK_SIZE; strm.next_out = out; rc = inflate(&strm, Z_NO_FLUSH); assert(rc != Z_STREAM_ERROR); // state not clobbered switch (rc) { case Z_NEED_DICT: rc = Z_DATA_ERROR; // and fall through case Z_DATA_ERROR: case Z_STREAM_ERROR: case Z_MEM_ERROR: free(in); inflateEnd(&strm); close_file(fs_, filename_); gzip_handle_error(rc, std::string("Error encountered during inflate with ")+filename_); return TILEDB_BF_ERR; } have = TILEDB_GZIP_CHUNK_SIZE - strm.avail_out; buffer_ = realloc(buffer_, buffer_size_+have); if (buffer_ == NULL) { BUFFER_ERROR_WITH_ERRNO("Cannot read to buffer; Mem allocation error"); return TILEDB_BF_ERR; } void *pmem = memcpy((char *)buffer_+buffer_size_, out, have); assert(pmem); buffer_size_ += have; if (strm.avail_in && strm.avail_out) { inflateEnd(&strm); // Start a new read for the start of a new segment size_t unprocessed_in = strm.avail_in; if (initialize_gzip_stream(&strm)) return TILEDB_BF_ERR; strm.avail_in = unprocessed_in; strm.next_in = in + (read_size-unprocessed_in); } } while (strm.avail_out == 0); } while (rc != Z_STREAM_END || processed < filesize_); // clean up before return free(in); inflateEnd(&strm); // All bytes have been decompressed assert(rc == Z_STREAM_END); assert(processed == filesize_); // adjust filesize_ to decompressed size as the reading will only be from memory now on filesize_ = buffer_size_; allocated_buffer_size_ = buffer_size_; return TILEDB_BF_OK; } int CompressedStorageBuffer::gzip_write_buffer() { unsigned have; z_stream strm; unsigned char out[TILEDB_GZIP_CHUNK_SIZE]; /* allocate deflate state */ strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; int rc = TILEDB_BF_OK; if ((rc = deflateInit2 (&strm, compression_level_, Z_DEFLATED, windowBits | GZIP_ENCODING, 8, Z_DEFAULT_STRATEGY)) != Z_OK) { gzip_handle_error(rc, std::string("Could not initialize compression for ")+filename_); return TILEDB_BF_ERR; } /* compress entire buffer */ strm.avail_in = buffer_size_; strm.next_in = (unsigned char *)buffer_; size_t processed = 0; do { strm.avail_out = TILEDB_GZIP_CHUNK_SIZE; strm.next_out = out; if ((rc = deflate(&strm, Z_FINISH)) == Z_STREAM_ERROR) { deflateEnd(&strm); BUFFER_PATH_ERROR("Encountered Z_STREAM_ERROR; Could not compress file", filename_); return TILEDB_UT_ERR; } have = TILEDB_GZIP_CHUNK_SIZE - strm.avail_out; if (compress_buffer_size_ < processed+have) { compress_buffer_ = realloc(compress_buffer_, processed+have); if (compress_buffer_ == NULL) { deflateEnd(&strm); BUFFER_ERROR_WITH_ERRNO("Cannot write to compress buffer; Mem allocation error"); return TILEDB_BF_ERR; } compress_buffer_size_ = processed+have; } void *pmem = memcpy((char *)compress_buffer_+processed, out, have); assert(pmem == (char *)compress_buffer_+processed); processed += have; } while (strm.avail_out == 0); deflateEnd(&strm); if (rc != Z_STREAM_END || strm.avail_in != 0) { // All input has not been compressed BUFFER_PATH_ERROR("All input could not be compressed: deflate error", filename_); return TILEDB_BF_ERR; } // Write directly if and only if the bytes to be written out is greater than the upload_file_size if (!fs_->get_upload_buffer_size() || (!compressed_write_buffer_size_ && processed >= fs_->get_upload_buffer_size())) { // Write directly if (fs_->write_to_file(filename_, compress_buffer_, processed)) { BUFFER_PATH_ERROR("Cannot write bytes", filename_); return TILEDB_BF_ERR; } } else { // Use another buffer to hold the compressed bytes until the minimum upload_file_size is satisfied if (!compressed_write_buffer_) { assert(compressed_write_buffer_size_ == 0); compressed_write_buffer_ = std::make_shared(fs_, filename_, fs_->get_upload_buffer_size()); } if (compressed_write_buffer_->append_buffer(compress_buffer_, processed)) { BUFFER_PATH_ERROR("Cannot write buffer after compression", filename_); return TILEDB_BF_ERR; } compressed_write_buffer_size_ += processed; if (compressed_write_buffer_size_ >= fs_->get_upload_buffer_size()) { compressed_write_buffer_->flush(); compressed_write_buffer_size_ = 0; } } return TILEDB_BF_OK; } int CompressedStorageBuffer::finalize() { int rc = TILEDB_BF_OK; if (!read_only_) { // Compress and write out any remaining bytes rc = write_buffer(); if (compressed_write_buffer_) { rc = rc || compressed_write_buffer_->finalize(); } if (rc) { BUFFER_PATH_ERROR("Could not finalize buffer after compression", filename_); } } return StorageBuffer::finalize() || rc; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_fs.cc000066400000000000000000000052251453617025200241770ustar00rootroot00000000000000/** * @file storage_fs.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Storage FS Base Implementation * */ #include "storage_fs.h" #include "utils.h" #include #include #include /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_fs_errmsg = ""; StorageFS::~StorageFS() { // Default } int StorageFS::close_file(const std::string& filename) { return TILEDB_FS_OK; } bool StorageFS::locking_support() { return false; } std::string StorageCloudFS::get_path(const std::string& path) { std::string pathname(path); if (path.find("://") != std::string::npos) { uri path_uri(path); pathname = path_uri.path(); if (pathname.empty()) { return ""; } } if (pathname[0] == '/') { return pathname.substr(1); } if (pathname.empty()) { return working_dir_; } else if (starts_with(pathname, working_dir_)) { // TODO: this is a hack for now, but should work fine with GenomicsDB. return pathname; } else { return append_paths(working_dir_, pathname); } } int StorageCloudFS::sync_path(const std::string& path) { // S3 and GCS allow for only write-once semantics, so committing the file will // happen only when the file is closed or when commit_file() is explicitly called // Forced write-once semantics for Azure Blob Storage too. return TILEDB_FS_OK; } int StorageCloudFS::close_file(const std::string& filename) { return commit_file(filename); } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_gcs.cc000066400000000000000000000262451453617025200243500ustar00rootroot00000000000000/** * @file storage_gcs.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018,2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * GCS Support for StorageFS */ #include "error.h" #include "storage_gcs.h" #include "storage_posixfs.h" #include "google/cloud/storage/client_options.h" #include #include #include #include #include #include #include #include #include #define GCS_ERROR(MSG, PATH) PATH_ERROR(TILEDB_FS_ERRMSG, "GCS: "+MSG, PATH, tiledb_fs_errmsg) #define GCS_ERROR1(MSG, STATUS, PATH) PATH_ERROR(TILEDB_FS_ERRMSG, "GCS: "+MSG+" "+STATUS.message(), PATH, tiledb_fs_errmsg) GCS::GCS(const std::string& home) { gcs_uri path_uri(home); if (path_uri.protocol().compare("gs") != 0) { throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "GCS FS only supports gs:// URI protocols"); } if (path_uri.bucket().size() == 0) { throw std::system_error(EPROTO, std::generic_category(), "GS URI does not seem to have a bucket specified"); } gcs::ChannelOptions channel_options; std::string ca_certs_location = locate_ca_certs(); if (!ca_certs_location.empty()) { channel_options.set_ssl_root_path(ca_certs_location); } auto client_options = gcs::ClientOptions::CreateDefaultClientOptions(channel_options); if (!client_options) { throw std::system_error(EIO, std::generic_category(), "Failed to create default GCS Client Options. "+ client_options.status().message()); } // Policy defaults are from /src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java. client_ = gcs::Client(*client_options, gcs::StrictIdempotencyPolicy(), gcs::AlwaysRetryIdempotencyPolicy(), gcs::LimitedErrorCountRetryPolicy(15), gcs::LimitedTimeRetryPolicy(std::chrono::milliseconds(4000000)), gcs::ExponentialBackoffPolicy(std::chrono::milliseconds(1000), std::chrono::milliseconds(256000), 2.0)); if (!client_) { throw std::system_error(EIO, std::generic_category(), "Failed to create GCS Client"+client_.status().message()); } StatusOr bucket_metadata = client_->GetBucketMetadata(path_uri.bucket()); if (!bucket_metadata) { throw std::system_error(EIO, std::generic_category(), "GCS FS only supports already existing buckets. Failed to locate bucket=" +path_uri.bucket()+" "+bucket_metadata.status().message()); } bucket_name_ = path_uri.bucket(); working_dir_ = get_path(path_uri.path()); // Set default buffer sizes, overridden with env vars TILEDB_DOWNLOAD_BUFFER_SIZE and TILEDB_UPLOAD_BUFFER_SIZE // Minimum size for each part of a multipart upload, except for the last part, using the same value for both uploads and downloads for now download_buffer_size_ = 5*1024*1024; // 5M upload_buffer_size_ = 5*1024*1024; // 5M } GCS::~GCS() { // Nothing yet } std::string GCS::current_dir() { return working_dir_; } int GCS::set_working_dir(const std::string& dir) { working_dir_ = get_path(dir); return TILEDB_FS_OK; } bool GCS::path_exists(const std::string& path) { return client_->GetObjectMetadata(bucket_name_, get_path(path)).status().ok(); } int GCS::create_path(const std::string& path) { const std::string EMPTY = ""; StatusOr object_metadata = client_->InsertObject(bucket_name_, get_path(path), std::move(EMPTY)); if (!object_metadata) { GCS_ERROR1("Error inserting object into bucket", object_metadata.status(), path); return TILEDB_FS_ERR; } else { return TILEDB_FS_OK; } } int GCS::delete_path(const std::string& path) { google::cloud::Status status = client_->DeleteObject(bucket_name_, get_path(path)); if (!status.ok()) { GCS_ERROR1("Could not delete path", status, path); return TILEDB_FS_ERR; } else { return TILEDB_FS_OK; } } std::string GCS::real_dir(const std::string& dir) { if (dir.find("://") != std::string::npos) { gcs_uri path_uri(dir); if (path_uri.bucket().compare(bucket_name_)) { throw std::runtime_error("Credentialed account during instantiation does not match the uri passed to real_dir. Aborting"); } } return get_path(dir); } int GCS::create_dir(const std::string& dir) { if (is_dir(dir) || is_file(dir)) { GCS_ERROR("Cannot create path as it already exists", dir); return TILEDB_FS_ERR; } return create_path(slashify(dir)); } int GCS::delete_dir(const std::string& dir) { if (is_file(dir)) { GCS_ERROR("Cannot delete dir as it seems to be a file", dir); return TILEDB_FS_ERR; } if (!is_dir(dir)) { GCS_ERROR("Cannot delete non-existent dir", dir); return TILEDB_FS_ERR; } gcs::DeleteByPrefix(*client_, bucket_name_, std::move(slashify(get_path(dir)))); return TILEDB_FS_OK; } std::vector GCS::get_dirs(const std::string& dir) { std::vector dirs; for (auto&& object : client_->ListObjectsAndPrefixes(bucket_name_, gcs::Prefix(slashify(get_path(dir))), gcs::Delimiter("/"))) { if (!object) { GCS_ERROR1("Error listing objects and prefixes", object.status(), dir); break; } auto result = *std::move(object); if (absl::holds_alternative(result)) { dirs.push_back(unslashify(absl::get(result))); } } return dirs; } std::vector GCS::get_files(const std::string& dir) { std::vector files; for (auto&& object : client_->ListObjects(bucket_name_, gcs::Prefix(slashify(get_path(dir))), gcs::Delimiter("/"))) { if (!object) { GCS_ERROR1("Error listing objects", object.status(), dir); break; } else { if (object->name().back() != '/') { files.push_back(object->name()); } } } return files; } int GCS::create_file(const std::string& filename, int flags, mode_t mode) { if (is_dir(filename) || is_file(filename)) { GCS_ERROR("Cannot create path as it already exists", filename); return TILEDB_FS_ERR; } return create_path(filename); } int GCS::delete_file(const std::string& filename) { if (!is_file(filename)) { GCS_ERROR("Cannot delete non-existent or non-file path", filename); return TILEDB_FS_ERR; } return delete_path(filename); } ssize_t GCS::file_size(const std::string& filename) { google::cloud::StatusOr object_metadata = client_->GetObjectMetadata(bucket_name_, get_path(filename)); if (object_metadata.ok()) { return object_metadata->size(); } else { return TILEDB_FS_ERR; } } int GCS::read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length) { if (length == 0) { return TILEDB_FS_OK; // Nothing to read } gcs::ObjectReadStream stream = client_->ReadObject(bucket_name_, get_path(filename), gcs::ReadRange(offset, offset+length)); if (!stream.status().ok()) { GCS_ERROR1("Failed to get object", stream.status(), filename); return TILEDB_FS_ERR; } stream.read(static_cast(buffer), length); if ((size_t)stream.gcount() < length) { GCS_ERROR("Could not read the file for bytes of length=" + std::to_string(length) + " from offset=" + std::to_string(offset), filename); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } #define CHUNK_SUFFIX "__tiledb__" int GCS::write_to_file(const std::string& filename, const void *buffer, size_t buffer_size) { if (buffer_size == 0) { return create_file(filename, 0, 0); } std::string filepath = get_path(filename); size_t part_number = 0; { // Separate block to limit mutex scope const std::lock_guard lock(write_map_mtx_); auto search = write_map_.find(filepath); if (search == write_map_.end()) { auto update_info = multipart_upload_info_t(); update_info.last_uploaded_size_ = buffer_size; write_map_.insert({filepath, std::move(update_info)}); } else { // https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload if (search->second.last_uploaded_size_ < 256*1024) { GCS_ERROR("Only the last of the uploadable parts can be less than 256KB", filepath); return TILEDB_FS_ERR; } part_number = ++search->second.part_number_; search->second.last_uploaded_size_ = buffer_size; } } std::string part = filepath + CHUNK_SUFFIX + std::to_string(part_number); std::string buf; buf.assign(static_cast(buffer), buffer_size); StatusOr object_metadata = client_->InsertObject(bucket_name_, part, std::move(buf)); if (!object_metadata) { GCS_ERROR1("Error writing part during InsertObject", object_metadata.status(), part); return TILEDB_FS_ERR; }; return TILEDB_FS_OK; } int GCS::commit_file(const std::string& filename) { std::string filepath = get_path(filename); std::vector parts; size_t num_parts = 0; { // Separate block to limit mutex scope const std::lock_guard lock(write_map_mtx_); auto found = write_map_.find(filepath); if (found == write_map_.end()) { return TILEDB_FS_OK; } num_parts = found->second.part_number_; write_map_.erase(filepath); } for(auto i=0ul; i<=num_parts; i++) { gcs::ComposeSourceObject part; part.object_name = filepath + CHUNK_SUFFIX + std::to_string(i); parts.emplace_back(std::move(part)); } std::string prefix = gcs::CreateRandomPrefixName(".tmpfiles"); StatusOr object_metadata = gcs::ComposeMany(*client_, bucket_name_, parts, prefix, filepath, /*ignore_cleanup_failures*/true); int rc = TILEDB_FS_OK; if (!object_metadata) { GCS_ERROR1("Error committing object during ComposeMany", object_metadata.status(), filepath); rc = TILEDB_FS_ERR; } // Cleanup parts and other temp files leftover by ComposeMany() gcs::DeleteByPrefix(*client_, bucket_name_, filepath+CHUNK_SUFFIX); gcs::DeleteByPrefix(*client_, bucket_name_, prefix); return rc; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_hdfs.cc000066400000000000000000000436501453617025200245170ustar00rootroot00000000000000/** * @file storage_hdfs.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 University of California, Los Angeles and Intel Corporation * @copyright Copyright (c) 2018-2019, 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * HDFS Support for StorageFS */ #ifdef USE_HDFS #include "storage_hdfs.h" #include "uri.h" #include "utils.h" #include #include #include #include #include #include #include #include #include #ifdef __APPLE__ #if !defined(ELIBACC) #define ELIBACC -1 #endif #endif #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_FS_ERRMSG << "hdfs: " << x << std::endl #else # define PRINT_ERROR(x) do { } while(0) #endif // Forward declarations static void invoke_function(hdfsFS hdfs_handle, std::unordered_map& map, int (*fn)(hdfsFS, hdfsFile, const std::string&)); static int sync_kernel(hdfsFS hdfs_handle, hdfsFile hdfs_file_handle, const std::string& path); static int close_kernel(hdfsFS hdfs_handle, hdfsFile hdfs_file_handle, const std::string& path); #include #include int get_rlimits(struct rlimit *limits) { if (getrlimit(RLIMIT_NOFILE, limits)) { PRINT_ERROR(std::string("Could not execute getrlimit ") + std::strerror(errno)); return -1; } return 0; } void maximize_rlimits() { struct rlimit limits; if (get_rlimits(&limits)) { return; } if (limits.rlim_cur == limits.rlim_max) { return; } limits.rlim_cur = limits.rlim_max; if (setrlimit(RLIMIT_NOFILE, &limits)) { PRINT_ERROR(std::string("Could not execute setrlimit ") + std::strerror(errno)); return; } get_rlimits(&limits); } hdfsFS hdfs_connect(uri path_uri, const std::string& name_node) { struct hdfsBuilder *builder = hdfsNewBuilder(); if (!builder) { PRINT_ERROR("Error getting hdfs builder"); throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "Error getting hdfs builder"); } // Forces the builder to always create a new instance of the FileSystem. // TODO: Figure out if cached instances of the builder can be used at least per thread, but // use a new instance for now. hdfsBuilderSetForceNewInstance(builder); hdfsBuilderSetNameNode(builder, name_node.c_str()); if (!path_uri.port().empty()) { hdfsBuilderSetNameNodePort(builder, path_uri.nport()); } return hdfsBuilderConnect(builder); } HDFS::HDFS(const std::string& home) { uri path_uri(home); std::string name_node; if (path_uri.host().empty()) { if (!path_uri.port().empty()) { PRINT_ERROR(std::string("home=") + home + " not supported. hdfs host and port have to be both empty"); throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "Home URI not supported: hdfs host and port have to be both empty"); } name_node.assign("default"); } else if (path_uri.protocol().compare("hdfs") != 0) { // s3/gs protocols name_node.assign(path_uri.protocol() + "://" + path_uri.host()); } else { if (path_uri.port().empty()) { PRINT_ERROR(std::string("home=") + home + " not supported. hdfs host and port have to be specified together"); throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "Home URI not supported: hdfs host and port have to be specified together"); } name_node.assign(path_uri.host()); } hdfs_handle_ = hdfs_connect(path_uri, name_node); if (!hdfs_handle_) { PRINT_ERROR("Error getting hdfs connection"); throw std::system_error(ECONNREFUSED, std::generic_category(), "Error getting hdfs connection"); } if (hdfsSetWorkingDirectory(hdfs_handle_, ((path_uri.path().empty())?(home+"/"):home).c_str())) { PRINT_ERROR(std::string("Error setting up hdfs working directory ") + home); throw std::system_error(ENOENT, std::generic_category(), "Error setting up hdfs working directory"); } // Maximize open file handle limits //maximize_rlimits(); } HDFS::~HDFS() { // Close any files that have been opened and not closed. invoke_function(hdfs_handle_, read_map_, &close_kernel); read_map_.clear(); invoke_function(hdfs_handle_, write_map_, &sync_kernel); invoke_function(hdfs_handle_, write_map_, &close_kernel); write_map_.clear(); hdfsDisconnect(hdfs_handle_); } static int print_errmsg(const std::string& errmsg) { if (errmsg.length() > 0) { PRINT_ERROR(errmsg); tiledb_fs_errmsg = TILEDB_FS_ERRMSG + errmsg; } return TILEDB_FS_ERR; } static hdfsFile get_hdfsFile(const std::string& filename, std::unordered_map& map) { auto search = map.find(filename); if (search != map.end()) { return search->second; } else { return NULL; } } static int close_read_hdfsFile(hdfsFS hdfs_handle, const std::string& filename, std::unordered_map& map, std::unordered_map& count_map, std::mutex& mtx) { int rc = TILEDB_FS_OK; mtx.lock(); hdfsFile hdfs_file_handle = get_hdfsFile(filename, map); if (hdfs_file_handle) { int count = 0; auto search = count_map.find(filename); if (search != count_map.end()) { count = search->second; } if (count == 0) { rc = close_kernel(hdfs_handle, hdfs_file_handle, filename); map.erase(filename); } } mtx.unlock(); return rc; } static int close_write_hdfsFile(hdfsFS hdfs_handle, const std::string& filename, std::unordered_map& map, std::mutex& mtx) { int rc = TILEDB_FS_OK; mtx.lock(); hdfsFile hdfs_file_handle = get_hdfsFile(filename, map); if (hdfs_file_handle) { sync_kernel(hdfs_handle, hdfs_file_handle, filename); rc = close_kernel(hdfs_handle, hdfs_file_handle, filename); } map.erase(filename); mtx.unlock(); return rc; } static void invoke_function(hdfsFS hdfs_handle, std::unordered_map& map, int (*fn)(hdfsFS, hdfsFile, const std::string&)) { for (auto it = map.begin(); it != map.end(); ++it) { std::string filename = it->first; hdfsFile hdfs_file_handle = it->second; // Call function kernel fn(hdfs_handle, hdfs_file_handle, filename); } } std::string HDFS::current_dir() { char working_dir[TILEDB_NAME_MAX_LEN]; if (hdfsGetWorkingDirectory(hdfs_handle_, working_dir, TILEDB_NAME_MAX_LEN) == NULL) { print_errmsg("Could not get current working dir"); return ""; } std::string cwd = working_dir; return working_dir; } int HDFS::set_working_dir(const std::string& dir) { if (hdfsSetWorkingDirectory(hdfs_handle_, dir.c_str())) { return print_errmsg(std::string("Error setting up hdfs working directory to ") + dir); } return TILEDB_FS_OK; } static bool is_path(const hdfsFS hdfs_handle, const char *path, const char kind) { if (!hdfsExists(hdfs_handle, path)) { hdfsFileInfo *file_info = hdfsGetPathInfo(hdfs_handle, path); if (file_info) { bool status = false; if ((char)(file_info->mKind) == kind) { status = true; } hdfsFreeFileInfo(file_info, 1); return status; } } return false; } bool HDFS::is_dir(const std::string& dir) { std::string slash("/"); if (dir.back() != '/') { return is_path(hdfs_handle_, (dir + slash).c_str(), 'D'); } return is_path(hdfs_handle_, dir.c_str(), 'D'); } bool HDFS::is_file(const std::string& file) { return is_path(hdfs_handle_, file.c_str(), 'F'); } std::string HDFS::real_dir(const std::string& dir) { if (dir.empty()) { return current_dir(); } else if (dir.find("://") != std::string::npos) { // absolute path return dir; } else if (starts_with(dir, "/")) { // seems to be an absolute path but without protocol/host information. return dir.substr(1); } else { // relative path return current_dir() + "/" + dir; } } int HDFS::create_dir(const std::string& dir) { if (is_dir(dir)) { return print_errmsg(std::string("Cannot create directory ") + dir + "; Directory already exists"); } if (hdfsCreateDirectory(hdfs_handle_, dir.c_str()) < 0) { return print_errmsg(std::string("Cannot create directory ") + dir); } return TILEDB_FS_OK; } int HDFS::delete_dir(const std::string& dir) { if (is_dir(dir)) { if (hdfsDelete(hdfs_handle_, dir.c_str(), 1) < 0) { return print_errmsg(std::string("Cannot delete directory ") + dir); } }else { return print_errmsg(std::string("Cannot delete path at ") + dir); } return TILEDB_FS_OK; } std::vector HDFS::get_dirs(const std::string& dir) { std::vector path_list; int num_entries = 0; hdfsFileInfo *file_info = hdfsListDirectory(hdfs_handle_, dir.c_str(), &num_entries); if (!file_info) { print_errmsg(std::string("Cannot list contents of dir ") + dir); } else { for (int i=0; i HDFS::get_files(const std::string& dir) { std::vector path_list; int num_entries = 0; hdfsFileInfo *file_info = hdfsListDirectory(hdfs_handle_, dir.c_str(), &num_entries); if (!file_info) { print_errmsg(std::string("Cannot list contents of dir ") + dir); } else { for (int i=0; imKind) != 'F') { print_errmsg(std::string("Cannot get file_size for path ") + filename + " that is not a file"); hdfsFreeFileInfo(file_info, 1); return TILEDB_FS_ERR; } size_t size = (size_t)file_info->mSize; hdfsFreeFileInfo(file_info, 1); return size; } static tSize max_tsize() { switch (sizeof(tSize)) { case 4: return INT32_MAX; default: print_errmsg("hdfs tSize width not recognized in read_from_file"); return 0; } } static int read_from_file_kernel(hdfsFS hdfs_handle, hdfsFile file, void* buffer, size_t length, off_t offset) { if (hdfsSeek(hdfs_handle, file, (tOffset)offset) < 0) { return print_errmsg(std::string("Cannot seek to offset in file")); } size_t max_bytes = max_tsize(); if (max_bytes == 0) { return TILEDB_FS_ERR; } size_t nbytes = 0; char *pbuf = (char *)buffer; do { tSize bytes_read = hdfsRead(hdfs_handle, file, (void *)pbuf, (length - nbytes) > max_bytes ? max_bytes : length - nbytes); if (bytes_read < 0) { return print_errmsg(std::string("Error reading file. ") + std::strerror(errno)); } nbytes += bytes_read; pbuf += bytes_read; } while (nbytes < length); return TILEDB_FS_OK; } static hdfsFile hdfs_open_file_for_read(hdfsFS hdfs_handle, const std::string& filename, size_t buffer_size) { return hdfsOpenFile(hdfs_handle, filename.c_str(), O_RDONLY, buffer_size, 0, 0); } // heuristic for io.file.buffer.size for good hdfs performance #define MAX_SIZE 16*1024*1024 static int read_count(const std::string& filename, std::unordered_map& read_count_map, bool incr) { int count = 0; auto search = read_count_map.find(filename); if (search != read_count_map.end()) { count = search->second; } if (incr) { ++count; } else { --count; } if (search != read_count_map.end()) { search->second = count; } else { read_count_map.emplace(filename, count); } return count; } int HDFS::read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length) { if (length == 0) { return TILEDB_FS_OK; // Nothing to read } // Not supporting simultaneous read/writes. if (get_hdfsFile(filename, write_map_)) { print_errmsg(std::string("File=") + filename + " is open simultaneously for reads/writes"); assert(false && "No support for simultaneous reads/writes"); } read_map_mtx_.lock(); hdfsFile file = get_hdfsFile(filename, read_map_); if (!file) { ssize_t size = file_size(filename); if (size == TILEDB_FS_ERR) { read_map_mtx_.unlock(); return TILEDB_FS_ERR; } file = hdfs_open_file_for_read(hdfs_handle_, filename, size>MAX_SIZE?MAX_SIZE:((size/getpagesize())+1)*getpagesize()); if (file) { read_map_.emplace(filename, file); } } int count = read_count(filename, read_count_, true); assert(count > 0 && "Read File Count cannot be less than 1"); read_map_mtx_.unlock(); if (!file) { return print_errmsg(std::string("Cannot open file ") + filename + " for read"); } int rc = read_from_file_kernel(hdfs_handle_, file, buffer, length, offset); read_map_mtx_.lock(); count = read_count(filename, read_count_, false); assert(count >= 0 && "Read File Count cannot be negative"); read_map_mtx_.unlock(); return rc; } static int write_to_file_kernel(hdfsFS hdfs_handle, hdfsFile file, const void* buffer, size_t buffer_size, size_t max_bytes) { size_t nbytes = 0; char *pbuf = (char *)buffer; do { tSize bytes_written = hdfsWrite(hdfs_handle, file, (void *)pbuf, (buffer_size - nbytes) > max_bytes ? max_bytes : buffer_size - nbytes); if (bytes_written < 0) { return print_errmsg(std::string("Error writing to file")); } nbytes += bytes_written; pbuf += bytes_written; } while (nbytes < buffer_size); if (hdfsFlush(hdfs_handle, file)) { return print_errmsg(std::string("Error flushing file ") + std::strerror(errno)); } return TILEDB_FS_OK; } int HDFS::write_to_file(const std::string& filename, const void *buffer, size_t buffer_size) { size_t max_bytes = max_tsize(); if (max_bytes == 0) { return TILEDB_FS_ERR; } hdfsFile file = get_hdfsFile(filename, write_map_); if (!file) { write_map_mtx_.lock(); file = hdfsOpenFile(hdfs_handle_, filename.c_str(), O_WRONLY, max_bytes, 0, 0); if (file) { write_map_.emplace(filename, file); } write_map_mtx_.unlock(); if (!file) { return print_errmsg(std::string("Cannot open file " + filename + " for write")); } } return write_to_file_kernel(hdfs_handle_, file, buffer, buffer_size, max_bytes); } int HDFS::move_path(const std::string& old_path, const std::string& new_path) { if (!hdfsExists(hdfs_handle_, new_path.c_str())) { return print_errmsg(std::string("Cannot move path ") + old_path + " to " + new_path + " as it exists"); } if (hdfsRename(hdfs_handle_, old_path.c_str(), new_path.c_str()) < 0) { return print_errmsg(std::string("Cannot rename path ") + old_path + " to " + new_path); } return TILEDB_FS_OK; } static int sync_kernel(hdfsFS hdfs_handle, hdfsFile hdfs_file_handle, const std::string& path) { if (hdfsHSync(hdfs_handle, hdfs_file_handle)) { return print_errmsg(std::string("Cannot sync file ") + path); } return TILEDB_FS_OK; } int HDFS::sync_path(const std::string& path) { int rc = TILEDB_FS_OK; auto search = write_map_.find(path); if(search != write_map_.end()) { rc = sync_kernel(hdfs_handle_, search->second, path); } return rc; } static int close_kernel(hdfsFS hdfs_handle, hdfsFile hdfs_file_handle, const std::string& filename) { if (hdfsCloseFile(hdfs_handle, hdfs_file_handle)) { return print_errmsg(std::string("Cannot close file ") + filename); } return TILEDB_FS_OK; } int HDFS::close_file(const std::string& filename) { int rc_close_read = TILEDB_FS_OK, rc_close_write=TILEDB_FS_OK; rc_close_read = close_read_hdfsFile(hdfs_handle_, filename, read_map_, read_count_, read_map_mtx_); rc_close_write = close_write_hdfsFile(hdfs_handle_, filename, write_map_, write_map_mtx_); if (rc_close_read) { return rc_close_read; } return rc_close_write; } bool HDFS::locking_support() { // No file locking available for distributed file systems return false; } #endif /* USE_HDFS */ genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_manager.cc000066400000000000000000001736131453617025200252100ustar00rootroot00000000000000/** * @file storage_manager.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the StorageManager class. */ #include "storage_manager.h" #include "uri.h" #include "utils.h" #include "storage_fs.h" #include #include #include #include #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_SM_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif #if defined HAVE_OPENMP && defined USE_PARALLEL_SORT #include #define SORT_LIB __gnu_parallel #else #include #define SORT_LIB std #endif #define SORT_2(first, last) SORT_LIB::sort((first), (last)) #define SORT_3(first, last, comp) SORT_LIB::sort((first), (last), (comp)) #define GET_MACRO(_1, _2, _3, NAME, ...) NAME #define SORT(...) GET_MACRO(__VA_ARGS__, SORT_3, SORT_2)(__VA_ARGS__) /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_sm_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ StorageManager::StorageManager() { } StorageManager::~StorageManager() { if (config_ != NULL) delete config_; } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int StorageManager::finalize() { return open_array_mtx_destroy(); } int StorageManager::init(StorageManagerConfig* config) { // Set configuration parameters if(config_set(config) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Initialize mutexes and return return open_array_mtx_init(); } StorageManagerConfig* StorageManager::get_config() { return config_; } /* ****************************** */ /* WORKSPACE */ /* ****************************** */ int StorageManager::workspace_create(const std::string& workspace) { // Check if the workspace is inside a workspace or another group std::string parent_dir = ::parent_dir(fs_, workspace); if(is_workspace(fs_, parent_dir) || is_group(fs_, parent_dir) || is_array(fs_, parent_dir) || is_metadata(fs_, parent_dir)) { std::string errmsg = "The workspace cannot be contained in another workspace, " "group, array or metadata directory"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Create workspace directory if(create_dir(fs_, workspace) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Create workspace file if(create_workspace_file(workspace) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } static std::vector list_workspaces(StorageFS *fs, const char *parent_dir) { std::vector workspace_dirs; std::vector all_dirs = ::get_dirs(fs, parent_dir); for(auto const& dir: all_dirs) { if(is_workspace(fs, dir)) { workspace_dirs.push_back(dir); } else if (fs->is_dir(dir) && !is_group(fs, dir) && !is_array(fs, dir) && !is_metadata(fs, dir)) { std::vector list = list_workspaces(fs, dir.c_str()); workspace_dirs.insert(std::end(workspace_dirs), std::begin(list), std::end(list)); } } return workspace_dirs; } static std::string relative_dir(std::string dir, const char *parent_dir) { if (dir.find(parent_dir) != std::string::npos && dir.size() > strlen(parent_dir)) { return dir.substr(strlen(parent_dir)+1); } else if (strstr(parent_dir, "://")) { uri path_uri(parent_dir); std::string path; if (path_uri.path().size() > 1) { if (path_uri.path()[0] == '/') { path = path_uri.path().substr(1); } else { path = path_uri.path(); } if (dir.find(path) != std::string::npos && dir.size() > path.size()) { if (path[path.size()-1] == '/') { return dir.substr(path.size()); } else { return dir.substr(path.size()+1); } } } } return dir; } int StorageManager::ls_workspaces( const char *parent_dir, char** workspaces, int& workspace_num) { std::vector workspace_dirs = list_workspaces(fs_, parent_dir); if (workspace_dirs.size() > static_cast::size_type>(workspace_num)) { std::string errmsg = "Cannot list TileDB workspaces; Directory buffer overflow"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } workspace_num = 0; for (auto dir: workspace_dirs) { strncpy(workspaces[workspace_num++], relative_dir(dir, parent_dir).c_str(), TILEDB_NAME_MAX_LEN); } // Success return TILEDB_SM_OK; } int StorageManager::ls_workspaces_c(const char* parent_dir, int& workspace_num) { // Get real parent directory std::string parent_dir_real = ::real_dir(fs_, parent_dir); // Initialize workspace number workspace_num = 0; std::vector dirs = ::get_dirs(fs_, parent_dir); for(auto const& dir: dirs) { if (is_workspace(fs_, dir)) { workspace_num++; } else if (fs_->is_dir(dir) && !is_group(fs_, dir) && !is_array(fs_, dir) && !is_metadata(fs_, dir)) { int num = 0; if (ls_workspaces_c(dir.c_str(), num) == TILEDB_SM_OK) { workspace_num += num; } } } // Success return TILEDB_SM_OK; } /* ****************************** */ /* GROUP */ /* ****************************** */ int StorageManager::group_create(const std::string& group) const { // Check if the group is inside a workspace or another group std::string parent_dir = ::parent_dir(fs_, group); if(!is_workspace(fs_, parent_dir) && !is_group(fs_, parent_dir)) { std::string errmsg = "The group must be contained in a workspace " "or another group"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Create group directory if(create_dir(fs_, group) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Create group file if(create_group_file(group) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } /* ****************************** */ /* ARRAY */ /* ****************************** */ int StorageManager::array_consolidate(const char* array_dir, size_t buffer_size, int batch_size) { // Create an array object Array* array; if(array_init( array, array_dir, TILEDB_ARRAY_CONSOLIDATE, NULL, NULL, 0) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Consolidate array Fragment* new_fragment; std::vector old_fragment_names; int rc_array_consolidate = array->consolidate(new_fragment, old_fragment_names, buffer_size, batch_size); // Close the array int rc_array_close = array_close(array->get_array_path_used()); // Finalize consolidation int rc_consolidation_finalize = consolidation_finalize(new_fragment, old_fragment_names); // Finalize array int rc_array_finalize = array->finalize(); delete array; int rc_delete = delete_directories(fs_, old_fragment_names); // Errors if(rc_array_consolidate != TILEDB_AR_OK) { tiledb_sm_errmsg = tiledb_ar_errmsg; return TILEDB_SM_ERR; } if(rc_array_close != TILEDB_SM_OK || rc_array_finalize != TILEDB_SM_OK || rc_consolidation_finalize != TILEDB_SM_OK || rc_delete != TILEDB_UT_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } int StorageManager::array_create(const ArraySchemaC* array_schema_c) const { // Initialize array schema ArraySchema* array_schema = new ArraySchema(fs_); if(array_schema->init(array_schema_c) != TILEDB_AS_OK) { delete array_schema; tiledb_sm_errmsg = tiledb_as_errmsg; return TILEDB_SM_ERR; } // Get real array directory name std::string dir = array_schema->array_name(); std::string parent_dir = ::parent_dir(fs_, dir); // Check if the array directory is contained in a workspace, group or array if(!is_workspace(fs_, parent_dir) && !is_group(fs_, parent_dir)) { std::string errmsg = std::string("Cannot create array; Directory '") + parent_dir + "' must be a TileDB workspace or group"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Create array with the new schema int rc = array_create(array_schema); // Clean up delete array_schema; // Return if(rc == TILEDB_SM_OK) return TILEDB_SM_OK; else return TILEDB_SM_ERR; } int StorageManager::array_create(const ArraySchema* array_schema) const { // Check array schema if(array_schema == NULL) { std::string errmsg = "Cannot create array; Empty array schema"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Create array directory std::string dir = array_schema->array_name(); if(create_dir(fs_, dir) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Store array schema if(array_store_schema(dir, array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Create consolidation filelock if(consolidation_filelock_create(dir) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } void StorageManager::array_get_fragment_names( const std::string& array, std::vector& fragment_names) { // Get directory names in the array folder fragment_names = get_fragment_dirs(fs_, real_dir(fs_, array)); // Sort the fragment names sort_fragment_names(fragment_names); } int StorageManager::array_load_book_keeping( const ArraySchema* array_schema, const std::vector& fragment_names, std::vector& book_keeping, int mode) { // For easy reference int fragment_num = fragment_names.size(); // Initialization book_keeping.resize(fragment_num); // Load the book-keeping for each fragment for(int i=0; iis_file(fs_->append_paths(fragment_names[i], std::string(TILEDB_COORDS) + TILEDB_FILE_SUFFIX)); // Create new book-keeping structure for the fragment BookKeeping* f_book_keeping = new BookKeeping( array_schema, dense, fragment_names[i], mode); // Load book-keeping if(f_book_keeping->load(fs_) != TILEDB_BK_OK) { delete f_book_keeping; tiledb_sm_errmsg = tiledb_bk_errmsg; return TILEDB_SM_ERR; } // Append to the open array entry book_keeping[i] = f_book_keeping; } // Success return TILEDB_SM_OK; } int StorageManager::array_load_schema( const char* array_dir, ArraySchema*& array_schema) const { // Get real array path std::string real_array_dir = ::real_dir(fs_, array_dir); // Check if array exists if(!is_array(fs_, real_array_dir)) { std::string errmsg = std::string("Cannot load array schema; Array '") + real_array_dir + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } std::string filename = fs_->append_paths(real_array_dir, TILEDB_ARRAY_SCHEMA_FILENAME); // Initialize buffer ssize_t buffer_size = file_size(fs_, filename); assert(buffer_size > 0); void* buffer = malloc(buffer_size); if (buffer == NULL) { std::string errmsg = "Storage Manager memory allocation error"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Load array schema if(read_from_file(fs_, filename, 0, buffer, buffer_size) == TILEDB_UT_ERR) { std::string errmsg = "Cannot load array schema; File reading error"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; free(buffer); return TILEDB_SM_ERR; } // Initialize array schema array_schema = new ArraySchema(fs_); if(array_schema->deserialize(buffer, buffer_size) != TILEDB_AS_OK) { free(buffer); delete array_schema; tiledb_sm_errmsg = tiledb_as_errmsg; return TILEDB_SM_ERR; } //Old TileDB version - create consolidate file if(!(array_schema->version_tag_exists())) { std::string filename = fs_->append_paths(real_array_dir, TILEDB_SM_CONSOLIDATION_FILELOCK_NAME); if (create_file(fs_, filename, O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) { std::string errmsg = "Cannot create consolidation file for old tiledb support"; tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } // Clean up close_file(fs_, filename); free(buffer); // Success return TILEDB_SM_OK; } int StorageManager::array_init( Array*& array, const char* array_dir, int mode, const void* subarray, const char** attributes, int attribute_num) { // Check array name length if(array_dir == NULL || strlen(array_dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid array name length"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Load array schema ArraySchema* array_schema; if(array_load_schema(array_dir, array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; std::string full_array_path = real_dir(fs_, array_dir); // Open the array OpenArray* open_array = NULL; if(array_read_mode(mode) || array_consolidate_mode(mode)) { if(array_open(full_array_path, open_array, mode) != TILEDB_SM_OK) return TILEDB_SM_ERR; } Array* array_clone = NULL; if (!array_consolidate_mode(mode)) { // Create the clone Array object. No need to clone for consolidation array_clone = new Array(); int rc_clone = array_clone->init( array_schema, full_array_path, open_array->fragment_names_, open_array->book_keeping_, mode, attributes, attribute_num, subarray, config_); // Handle error if(rc_clone != TILEDB_AR_OK) { delete array_schema; delete array_clone; array = NULL; if(array_read_mode(mode)) array_close(full_array_path); tiledb_sm_errmsg = tiledb_ar_errmsg; return TILEDB_SM_ERR; } } // Create actual array array = new Array(); int rc = array->init( array_schema, full_array_path, open_array->fragment_names_, open_array->book_keeping_, mode, attributes, attribute_num, subarray, config_, array_clone); // Handle error if(rc != TILEDB_AR_OK) { delete array_schema; delete array; array = NULL; if(array_read_mode(mode)) array_close(full_array_path); tiledb_sm_errmsg = tiledb_ar_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_finalize(Array* array) { // If the array is NULL, do nothing if(array == NULL) return TILEDB_SM_OK; // Finalize and close the array int rc_finalize = array->finalize(); int rc_close = TILEDB_SM_OK; if(array->read_mode()) rc_close = array_close(array->get_array_path_used()); // Clean up delete array; // Errors if(rc_close != TILEDB_SM_OK) return TILEDB_SM_ERR; if(rc_finalize != TILEDB_AR_OK) { tiledb_sm_errmsg = tiledb_ar_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_sync(Array* array) { // If the array is NULL, do nothing if(array == NULL) return TILEDB_SM_OK; // Sync array if(array->sync() != TILEDB_AR_OK) { tiledb_sm_errmsg = tiledb_ar_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_sync_attribute( Array* array, const std::string& attribute) { // If the array is NULL, do nothing if(array == NULL) return TILEDB_SM_OK; // Sync array if(array->sync_attribute(attribute) != TILEDB_AR_OK) { tiledb_sm_errmsg = tiledb_ar_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_iterator_init( ArrayIterator*& array_it, const char* array_dir, int mode, const void* subarray, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes, const char* filter_expression) { // Create Array object. This also creates/updates an open array entry Array* array; if(array_init( array, array_dir, mode, subarray, attributes, attribute_num) != TILEDB_SM_OK) { array_it = NULL; return TILEDB_SM_ERR; } // Create ArrayIterator object array_it = new ArrayIterator(); if(array_it->init(array, buffers, buffer_sizes, filter_expression) != TILEDB_AIT_OK) { array_finalize(array); delete array_it; array_it = NULL; tiledb_sm_errmsg = tiledb_ait_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_iterator_finalize( ArrayIterator* array_it) { // If the array iterator is NULL, do nothing if(array_it == NULL) return TILEDB_SM_OK; // Finalize and close array std::string array_name = array_it->array_name(); int rc_finalize = array_it->finalize(); int rc_close = array_close(array_name); // Clean up delete array_it; // Errors if(rc_finalize != TILEDB_AIT_OK) { tiledb_sm_errmsg = tiledb_ait_errmsg; return TILEDB_SM_ERR; } if(rc_close != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } /* ****************************** */ /* METADATA */ /* ****************************** */ int StorageManager::metadata_consolidate(const char* metadata_dir) { // Load metadata schema ArraySchema* array_schema; if(metadata_load_schema(metadata_dir, array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Set attributes char** attributes; int attribute_num = array_schema->attribute_num(); attributes = new char*[attribute_num+1]; for(int i=0; iattribute(i).c_str(); size_t attribute_len = strlen(attribute); attributes[i] = new char[attribute_len+1]; strcpy(attributes[i], attribute); } // Create a metadata object Metadata* metadata; int rc_init = metadata_init( metadata, metadata_dir, TILEDB_METADATA_READ, (const char**) attributes, attribute_num+1); // Clean up for(int i=0; i old_fragment_names; int rc_metadata_consolidate = metadata->consolidate(new_fragment, old_fragment_names); // Close the underlying array std::string array_name = metadata->array_schema()->array_name(); int rc_array_close = array_close(array_name); // Finalize consolidation int rc_consolidation_finalize = consolidation_finalize(new_fragment, old_fragment_names); // Finalize metadata int rc_metadata_finalize = metadata->finalize(); delete metadata; int rc_delete = delete_directories(fs_, old_fragment_names); // Errors if(rc_metadata_consolidate != TILEDB_MT_OK) { tiledb_sm_errmsg = tiledb_mt_errmsg; return TILEDB_SM_ERR; } if(rc_array_close != TILEDB_SM_OK || rc_metadata_finalize != TILEDB_SM_OK || rc_consolidation_finalize != TILEDB_SM_OK || rc_delete != TILEDB_UT_OK ) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } int StorageManager::metadata_create( const MetadataSchemaC* metadata_schema_c) const { // Initialize array schema ArraySchema* array_schema = new ArraySchema(fs_); if(array_schema->init(metadata_schema_c) != TILEDB_AS_OK) { delete array_schema; tiledb_sm_errmsg = tiledb_as_errmsg; return TILEDB_SM_ERR; } // Get real array directory name std::string dir = array_schema->array_name(); std::string parent_dir = ::parent_dir(fs_, dir); // Check if the array directory is contained in a workspace, group or array if(!is_workspace(fs_, parent_dir) && !is_group(fs_, parent_dir) && !is_array(fs_, parent_dir)) { std::string errmsg = std::string("Cannot create metadata; Directory '") + parent_dir + "' must be a TileDB workspace, group, or array"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Create array with the new schema int rc = metadata_create(array_schema); // Clean up delete array_schema; // Return if(rc == TILEDB_SM_OK) return TILEDB_SM_OK; else return TILEDB_SM_ERR; } int StorageManager::metadata_create(const ArraySchema* array_schema) const { // Check metadata schema if(array_schema == NULL) { std::string errmsg = "Cannot create metadata; Empty metadata schema"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Create array directory std::string dir = array_schema->array_name(); if(create_dir(fs_, dir) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } std::string filename = fs_->append_paths(dir, TILEDB_METADATA_SCHEMA_FILENAME); // Serialize metadata schema void* array_schema_bin; size_t array_schema_bin_size; if(array_schema->serialize(array_schema_bin, array_schema_bin_size) != TILEDB_AS_OK) { tiledb_sm_errmsg = tiledb_as_errmsg; return TILEDB_SM_ERR; } // Store the array schema int rc = write_to_file(fs_, filename, array_schema_bin, array_schema_bin_size); if (!rc) { rc = close_file(fs_, filename); } if (rc) { free(array_schema_bin); std::string errmsg = std::string("Cannot create metadata"); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Clean up free(array_schema_bin); // Create consolidation filelock if(consolidation_filelock_create(dir) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } int StorageManager::metadata_load_schema( const char* metadata_dir, ArraySchema*& array_schema) const { // Get real array path std::string real_metadata_dir = ::real_dir(fs_, metadata_dir); // Check if metadata exists if(!is_metadata(fs_, real_metadata_dir)) { PRINT_ERROR(std::string("Cannot load metadata schema; Metadata '") + real_metadata_dir + "' does not exist"); return TILEDB_SM_ERR; } // Open array schema file std::string filename = fs_->append_paths(real_metadata_dir, TILEDB_METADATA_SCHEMA_FILENAME); // Initialize buffer ssize_t buffer_size = file_size(fs_, filename); if(buffer_size == 0) { std::string errmsg = "Cannot load metadata schema; Empty metadata schema file"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } void* buffer = malloc(buffer_size); // Load array schema if (read_from_file(fs_, filename, 0, buffer, buffer_size) == TILEDB_UT_ERR) { free(buffer); std::string errmsg = "Cannot load metadata schema; File reading error\n" + tiledb_ut_errmsg; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Initialize array schema array_schema = new ArraySchema(fs_); if(array_schema->deserialize(buffer, buffer_size) == TILEDB_AS_ERR) { free(buffer); delete array_schema; tiledb_sm_errmsg = tiledb_as_errmsg; return TILEDB_SM_ERR; } // Clean up close_file(fs_, filename); free(buffer); // Success return TILEDB_SM_OK; } int StorageManager::metadata_init( Metadata*& metadata, const char* metadata_dir, int mode, const char** attributes, int attribute_num) { // Check metadata name length if(metadata_dir == NULL || strlen(metadata_dir) > TILEDB_NAME_MAX_LEN) { std::string errmsg = "Invalid metadata name length"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Load metadata schema ArraySchema* array_schema; if(metadata_load_schema(metadata_dir, array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Open the array that implements the metadata OpenArray* open_array = NULL; if(mode == TILEDB_METADATA_READ) { if(array_open( real_dir(fs_, metadata_dir), open_array, TILEDB_ARRAY_READ) != TILEDB_SM_OK) return TILEDB_SM_ERR; } // Create metadata object metadata = new Metadata(); int rc = metadata->init( array_schema, open_array->fragment_names_, open_array->book_keeping_, mode, attributes, attribute_num, config_); // Return if(rc != TILEDB_MT_OK) { delete array_schema; delete metadata; metadata = NULL; array_close(metadata_dir); tiledb_sm_errmsg = tiledb_mt_errmsg; return TILEDB_SM_ERR; } else { return TILEDB_SM_OK; } } int StorageManager::metadata_finalize(Metadata* metadata) { // If the metadata is NULL, do nothing if(metadata == NULL) return TILEDB_SM_OK; // Finalize the metadata and close the underlying array std::string array_name = metadata->array_schema()->array_name(); int mode = metadata->array()->mode(); int rc_finalize = metadata->finalize(); int rc_close = TILEDB_SM_OK; if(mode == TILEDB_METADATA_READ) rc_close = array_close(array_name); // Clean up delete metadata; // Errors if(rc_close != TILEDB_SM_OK) return TILEDB_SM_ERR; if(rc_finalize != TILEDB_MT_OK) { tiledb_sm_errmsg = tiledb_mt_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::metadata_iterator_init( MetadataIterator*& metadata_it, const char* metadata_dir, const char** attributes, int attribute_num, void** buffers, size_t* buffer_sizes) { // Create metadata object Metadata* metadata; if(metadata_init( metadata, metadata_dir, TILEDB_METADATA_READ, attributes, attribute_num) != TILEDB_SM_OK) { metadata_it = NULL; return TILEDB_SM_ERR; } // Create MetadataIterator object metadata_it = new MetadataIterator(); if(metadata_it->init(metadata, buffers, buffer_sizes) != TILEDB_MIT_OK) { metadata_finalize(metadata); delete metadata_it; metadata_it = NULL; tiledb_sm_errmsg = tiledb_mit_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::metadata_iterator_finalize( MetadataIterator* metadata_it) { // If the metadata iterator is NULL, do nothing if(metadata_it == NULL) return TILEDB_SM_OK; // Close array and finalize metadata std::string metadata_name = metadata_it->metadata_name(); int rc_finalize = metadata_it->finalize(); int rc_close = array_close(metadata_name); // Clean up delete metadata_it; // Errors if(rc_finalize != TILEDB_MIT_OK) { tiledb_sm_errmsg = tiledb_mit_errmsg; return TILEDB_SM_ERR; } if(rc_close != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } /* ****************************** */ /* MISC */ /* ****************************** */ int StorageManager::ls( const char* parent_dir, char** dirs, int* dir_types, int& dir_num) const { // Initialize directory counter int dir_i = 0; int dir_type; std::vector all_dirs = ::get_dirs(fs_, parent_dir); for(auto const& dir: all_dirs) { if(is_workspace(fs_, dir)) { dir_type = TILEDB_WORKSPACE; } else if(is_group(fs_, dir)) { dir_type = TILEDB_GROUP; } else if(is_metadata(fs_, dir)) { dir_type = TILEDB_METADATA; } else if(is_array(fs_, dir)){ dir_type = TILEDB_ARRAY; } else { dir_type = -1; } if (dir_type >= 0) { if (dir_i < dir_num) { strncpy(dirs[dir_i], relative_dir(dir, parent_dir).c_str(), TILEDB_NAME_MAX_LEN); dir_types[dir_i++] = dir_type; } else { std::string errmsg = "Cannot list entire TileDB directory; Directory buffer overflow"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } } // Set the number of directories dir_num = dir_i; // Success return TILEDB_SM_OK; } int StorageManager::ls_c(const char* parent_dir, int& dir_num) const { // Get real parent directory std::string parent_dir_real = ::real_dir(fs_, parent_dir); // Initialize directory number dir_num =0; std::vector dirs = ::get_dirs(fs_, parent_dir); for(auto const& dir: dirs) { if(is_workspace(fs_, dir) || is_group(fs_, dir) || is_metadata(fs_, dir) || is_array(fs_, dir)){ dir_num++; } } // Success return TILEDB_SM_OK; } int StorageManager::clear(const std::string& dir) const { if(is_workspace(fs_, dir)) { return workspace_clear(dir); } else if(is_group(fs_, dir)) { return group_clear(dir); } else if(is_array(fs_, dir)) { return array_clear(dir); } else if(is_metadata(fs_, dir)) { return metadata_clear(dir); } else { std::string errmsg = "Clear failed; Invalid directory"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } int StorageManager::delete_entire(const std::string& dir) { if(is_workspace(fs_, dir)) { return workspace_delete(dir); } else if(is_group(fs_, dir)) { return group_delete(dir); } else if(is_array(fs_, dir)) { return array_delete(dir); } else if(is_metadata(fs_, dir)) { return metadata_delete(dir); } else { std::string errmsg = "Delete failed; Invalid directory"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::move( const std::string& old_dir, const std::string& new_dir) { if(is_workspace(fs_, old_dir)) { return workspace_move(old_dir, new_dir); } else if(is_group(fs_, old_dir)) { return group_move(old_dir, new_dir); } else if(is_array(fs_, old_dir)) { return array_move(old_dir, new_dir); } else if(is_metadata(fs_, old_dir)) { return metadata_move(old_dir, new_dir); } else { std::string errmsg = "Move failed; Invalid source directory"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ int StorageManager::array_clear( const std::string& array) const { // Get real array directory name std::string array_real = ::real_dir(fs_, array); // Check if the array exists if(!is_array(fs_, array_real)) { std::string errmsg = std::string("Array '") + array_real + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } std::vector all_dirs = ::get_dirs(fs_, array_real); for(auto const& dir: all_dirs) { if(is_metadata(fs_, dir)) { metadata_delete(dir); } else if(is_fragment(fs_, dir)) { delete_dir(fs_, dir); } else { std::string errmsg = std::string("Cannot delete non TileDB related element '") + dir + "'"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } // Success return TILEDB_SM_OK; } int StorageManager::array_close(const std::string& array) { // Lock mutexes if(open_array_mtx_lock() != TILEDB_SM_OK) return TILEDB_SM_ERR; // Find the open array entry std::map::iterator it = open_arrays_.find(real_dir(fs_, array)); // Sanity check if(it == open_arrays_.end()) { std::string errmsg = "Cannot close array; Open array entry not found"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Lock the mutex of the array if(it->second->mutex_lock() != TILEDB_SM_OK) return TILEDB_SM_ERR; // Decrement counter --(it->second->cnt_); // Delete open array entry if necessary int rc_mtx_destroy = TILEDB_SM_OK; int rc_filelock = TILEDB_SM_OK; if(it->second->cnt_ == 0) { // Clean up book-keeping std::vector::iterator bit = it->second->book_keeping_.begin(); for(; bit != it->second->book_keeping_.end(); ++bit) delete *bit; // Unlock and destroy mutexes it->second->mutex_unlock(); rc_mtx_destroy = it->second->mutex_destroy(); // Unlock consolidation filelock rc_filelock = consolidation_filelock_unlock( it->second->consolidation_filelock_); // Delete array schema if(it->second->array_schema_ != NULL) delete it->second->array_schema_; // Free open array delete it->second; // Delete open array entry open_arrays_.erase(it); } else { // Unlock the mutex of the array if(it->second->mutex_unlock() != TILEDB_SM_OK) return TILEDB_SM_ERR; } // Unlock mutexes int rc_mtx_unlock = open_array_mtx_unlock(); // Return if(rc_mtx_destroy != TILEDB_SM_OK || rc_filelock != TILEDB_SM_OK || rc_mtx_unlock != TILEDB_SM_OK) return TILEDB_SM_ERR; else return TILEDB_SM_OK; } int StorageManager::array_delete( const std::string& array) const { // Clear the array if(array_clear(array) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Delete array directory if(delete_dir(fs_, array) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_get_open_array_entry( const std::string& array, OpenArray*& open_array, bool& opened_first_time) { // Lock mutexes if(open_array_mtx_lock() != TILEDB_SM_OK) return TILEDB_SM_ERR; // Find the open array entry std::map::iterator it = open_arrays_.find(array); // Create and init entry if it does not exist if(it == open_arrays_.end()) { open_array = new OpenArray(); open_array->cnt_ = 0; open_array->consolidation_filelock_ = -1; open_array->book_keeping_ = std::vector(); if(open_array->mutex_init() != TILEDB_SM_OK) { open_array->mutex_unlock(); return TILEDB_SM_ERR; } open_arrays_[array] = open_array; opened_first_time = true; } else { open_array = it->second; opened_first_time = false; } // Increment counter ++(open_array->cnt_); // Unlock mutexes if(open_array_mtx_unlock() != TILEDB_SM_OK) { --open_array->cnt_; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_move( const std::string& old_array, const std::string& new_array) const { // Get real array directory name std::string old_array_real = real_dir(fs_, old_array); std::string new_array_real = real_dir(fs_, new_array); // Check if the old array exists if(!is_array(fs_, old_array_real)) { std::string errmsg = std::string("Array '") + old_array_real + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Make sure that the new array is not an existing directory if(is_dir(fs_, new_array_real)) { std::string errmsg = std::string("Directory '") + new_array_real + "' already exists"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Check if the new array are inside a workspace or group std::string new_array_parent_folder = parent_dir(fs_, new_array_real); if(!is_group(fs_, new_array_parent_folder) && !is_workspace(fs_, new_array_parent_folder)) { std::string errmsg = std::string("Folder '") + new_array_parent_folder + "' must " "be either a workspace or a group"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Rename array if(move_path(fs_, old_array_real, new_array_real)) { std::string errmsg = std::string("Cannot move array; ") + strerror(errno); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Incorporate new name in the array schema ArraySchema* array_schema; if(array_load_schema(new_array_real.c_str(), array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; array_schema->set_array_name(new_array_real.c_str()); // Store the new schema if(array_store_schema(new_array_real, array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Clean up delete array_schema; // Success return TILEDB_SM_OK; } int StorageManager::array_open( const std::string& array_name, OpenArray*& open_array, int mode) { auto opened_first_time = false; // Get the open array entry if(array_get_open_array_entry(array_name, open_array, opened_first_time) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Lock the mutex of the array if(open_array->mutex_lock() != TILEDB_SM_OK) return TILEDB_SM_ERR; // First time the array is opened if(opened_first_time) { // Acquire shared lock on consolidation filelock if(consolidation_filelock_lock( array_name, open_array->consolidation_filelock_, TILEDB_SM_SHARED_LOCK) != TILEDB_SM_OK) { open_array->mutex_unlock(); return TILEDB_SM_ERR; } // Get the fragment names array_get_fragment_names(array_name, open_array->fragment_names_); // Get array schema if(is_array(fs_, array_name)) { // Array if(array_load_schema( array_name.c_str(), open_array->array_schema_) != TILEDB_SM_OK) return TILEDB_SM_ERR; } else { // Metadata if(metadata_load_schema( array_name.c_str(), open_array->array_schema_) != TILEDB_SM_OK) return TILEDB_SM_ERR; } } if (!array_consolidate_mode(mode)) { // Load the book-keeping for each fragment. For consolidate mode, each fragment will be opened // separately if(array_load_book_keeping( open_array->array_schema_, open_array->fragment_names_, open_array->book_keeping_, mode) != TILEDB_SM_OK) { delete open_array->array_schema_; open_array->array_schema_ = NULL; open_array->mutex_unlock(); return TILEDB_SM_ERR; } } // Unlock the mutex of the array if(open_array->mutex_unlock() != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::array_store_schema( const std::string& dir, const ArraySchema* array_schema) const { // Array schema file std::string filename = fs_->append_paths(dir, TILEDB_ARRAY_SCHEMA_FILENAME); if (is_file(fs_, filename) && delete_file(fs_, filename) == TILEDB_UT_ERR) { std::string errmsg = std::string("Cannot store schema as existing file cannot be deleted"); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Serialize array schema void* array_schema_bin; size_t array_schema_bin_size; if(array_schema->serialize(array_schema_bin, array_schema_bin_size) != TILEDB_AS_OK) { tiledb_sm_errmsg = tiledb_as_errmsg; return TILEDB_SM_ERR; } // Store the array schema int rc = write_to_file(fs_, filename, array_schema_bin, array_schema_bin_size); if (!rc) { rc = close_file(fs_, filename); } if (rc) { free(array_schema_bin); std::string errmsg = std::string("Cannot store schema"); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Clean up free(array_schema_bin); // Success return TILEDB_SM_OK; } int StorageManager::config_set(StorageManagerConfig* config) { // Store config locally config_ = config; fs_ = config_->get_filesystem(); // Success return TILEDB_SM_OK; } int StorageManager::consolidation_filelock_create( const std::string& dir) const { // Create file std::string filename = fs_->append_paths(dir, TILEDB_SM_CONSOLIDATION_FILELOCK_NAME); if (create_file(fs_, filename, O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) { std::string errmsg = std::string("Cannot create consolidation filelock"); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::consolidation_filelock_lock( const std::string& array_name, int& fd, int lock_type) const { if (!fs_->locking_support()) { return TILEDB_SM_OK; } // Prepare the flock struct struct flock fl; if(lock_type == TILEDB_SM_SHARED_LOCK) { fl.l_type = F_RDLCK; } else if(lock_type == TILEDB_SM_EXCLUSIVE_LOCK) { fl.l_type = F_WRLCK; } else { std::string errmsg = "Cannot lock consolidation filelock; Invalid lock type"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } fl.l_whence = SEEK_SET; fl.l_start = 0; fl.l_len = 0; fl.l_pid = getpid(); // Prepare the filelock name std::string array_name_real = real_dir(fs_, array_name); std::string filename = fs_->append_paths(array_name_real, TILEDB_SM_CONSOLIDATION_FILELOCK_NAME); // Create consolidation lock file if necessary if (!fs_->is_file(filename)) { if (consolidation_filelock_create(array_name_real)) { std::string errmsg = std::string("Cannot lock consolidation filelock; consolidation lock file doesn't exist and ") + " cannot create consolidation lock file "+filename; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } // Open the file fd = ::open(filename.c_str(), (lock_type == TILEDB_SM_SHARED_LOCK) ? O_RDONLY : O_RDWR); if(fd == -1) { std::string errmsg = "Cannot lock consolidation filelock; Cannot open filelock"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Acquire the lock if(fcntl(fd, F_SETLKW, &fl) == -1) { std::string errmsg = "Cannot lock consolidation filelock; Cannot lock"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::consolidation_filelock_unlock(int fd) const { if (!fs_->locking_support()) { return TILEDB_SM_OK; } if(::close(fd) == -1) { std::string errmsg = "Cannot unlock consolidation filelock; Cannot close filelock"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } else { return TILEDB_SM_OK; } } int StorageManager::consolidation_finalize( Fragment* new_fragment, const std::vector& old_fragment_names) { // Trivial case - there was no consolidation if(old_fragment_names.size() == 0) return TILEDB_SM_OK; // Acquire exclusive lock on consolidation filelock int fd; if(consolidation_filelock_lock( new_fragment->array()->get_array_path_used(), fd, TILEDB_SM_EXCLUSIVE_LOCK) != TILEDB_SM_OK) { delete new_fragment; return TILEDB_SM_ERR; } // Finalize new fragment - makes the new fragment visible to new reads int rc = new_fragment->finalize(); delete new_fragment; if(rc != TILEDB_FG_OK) { tiledb_sm_errmsg = tiledb_fg_errmsg; return TILEDB_SM_ERR; } // Make old fragments invisible to new reads int fragment_num = old_fragment_names.size(); for(int i=0; iappend_paths(old_fragment_names[i], TILEDB_FRAGMENT_FILENAME); if(delete_file(fs_, old_fragment_filename)) { std::string errmsg = std::string("Cannot remove fragment file during " "finalizing consolidation; ") + strerror(errno); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } // Unlock consolidation filelock if(consolidation_filelock_unlock(fd) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Success return TILEDB_SM_OK; } int StorageManager::create_group_file(const std::string& group) const { // Create file std::string filename = fs_->append_paths(group, TILEDB_GROUP_FILENAME); if(create_file(fs_, filename, O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_FS_ERR) { std::string errmsg = std::string("Failed to create group file\n") + tiledb_ut_errmsg; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::create_workspace_file(const std::string& workspace) const { // Create file std::string filename = fs_->append_paths(workspace, TILEDB_WORKSPACE_FILENAME); if(create_file(fs_, filename, O_WRONLY | O_CREAT | O_SYNC, S_IRWXU) == TILEDB_UT_ERR) { std::string errmsg = std::string("Failed to create workspace file\n") + tiledb_ut_errmsg; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::group_clear( const std::string& group) const { // Get real group path std::string group_real = ::real_dir(fs_, group); // Check if group exists if(!is_group(fs_, group_real)) { std::string errmsg = std::string("Group '") + group_real + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Do not delete if it is a workspace if(is_workspace(fs_, group_real)) { std::string errmsg = std::string("Group '") + group_real + "' is also a workspace"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Delete all groups, arrays and metadata inside the group directory std::vector all_dirs = ::get_dirs(fs_, group_real); for(auto const& dir: all_dirs) { if(is_group(fs_, dir)) { group_delete(dir); } else if(is_metadata(fs_, dir)) { metadata_delete(dir); } else if(is_array(fs_, dir)){ array_delete(dir); } else { // Non TileDB related std::string errmsg = std::string("Cannot delete non TileDB related element '") + dir + "'"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } // Success return TILEDB_SM_OK; } int StorageManager::group_delete( const std::string& group) const { // Clear the group if(group_clear(group) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Delete group directory if(delete_dir(fs_, group) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::group_move( const std::string& old_group, const std::string& new_group) const { // Get real group directory names std::string old_group_real = real_dir(fs_, old_group); std::string new_group_real = real_dir(fs_, new_group); // Check if the old group is also a workspace if(is_workspace(fs_, old_group_real)) { std::string errmsg = std::string("Group '") + old_group_real + "' is also a workspace"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Check if the old group exists if(!is_group(fs_, old_group_real)) { std::string errmsg = std::string("Group '") + old_group_real + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Make sure that the new group is not an existing directory if(is_dir(fs_, new_group_real)) { std::string errmsg = std::string("Directory '") + new_group_real + "' already exists"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Check if the new group is inside a workspace or group std::string new_group_parent_folder = parent_dir(fs_, new_group_real); if(!is_group(fs_, new_group_parent_folder) && !is_workspace(fs_, new_group_parent_folder)) { std::string errmsg = std::string("Folder '") + new_group_parent_folder + "' must " "be either a workspace or a group"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Rename if(move_path(fs_, old_group_real, new_group_real)) { std::string errmsg = std::string("Cannot move group\n") + tiledb_ut_errmsg; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::metadata_clear( const std::string& metadata) const { // Get real metadata directory name std::string metadata_real = ::real_dir(fs_, metadata); // Check if the metadata exists if(!is_metadata(fs_, metadata_real)) { std::string errmsg = std::string("Metadata '") + metadata_real + "' do not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } std::vector all_dirs = ::get_dirs(fs_, metadata_real); for(auto const& dir: all_dirs) { if(is_fragment(fs_, dir)) { delete_dir(fs_, dir); } else { std::string errmsg = std::string("Cannot delete non TileDB related element '") + dir + "'"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } return TILEDB_SM_OK; } int StorageManager::metadata_delete( const std::string& metadata) const { // Get real metadata directory name std::string metadata_real = ::real_dir(fs_, metadata); // Clear the metadata if(metadata_clear(metadata_real) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Delete metadata directory if(delete_dir(fs_, metadata_real) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::metadata_move( const std::string& old_metadata, const std::string& new_metadata) const { // Get real metadata directory name std::string old_metadata_real = real_dir(fs_, old_metadata); std::string new_metadata_real = real_dir(fs_, new_metadata); // Check if the old metadata exists if(!is_metadata(fs_, old_metadata_real)) { std::string errmsg = std::string("Metadata '") + old_metadata_real + "' do not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Make sure that the new metadata is not an existing directory if(is_dir(fs_, new_metadata_real)) { std::string errmsg = std::string("Directory '") + new_metadata_real + "' already exists"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Check if the new metadata are inside a workspace, group or array std::string new_metadata_parent_folder = parent_dir(fs_, new_metadata_real); if(!is_group(fs_, new_metadata_parent_folder) && !is_workspace(fs_, new_metadata_parent_folder) && !is_array(fs_, new_metadata_parent_folder)) { std::string errmsg = std::string("Folder '") + new_metadata_parent_folder + "' must " "be workspace, group or array"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Rename metadata if(move_path(fs_, old_metadata_real, new_metadata_real)) { std::string errmsg = std::string("Cannot move metadata; ") + strerror(errno); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Incorporate new name in the array schema ArraySchema* array_schema; if(array_load_schema(new_metadata_real.c_str(), array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; array_schema->set_array_name(new_metadata_real.c_str()); // Store the new schema if(array_store_schema(new_metadata_real, array_schema) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Clean up delete array_schema; // Success return TILEDB_SM_OK; } int StorageManager::open_array_mtx_destroy() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_destroy(&open_array_omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_destroy(&open_array_pthread_mtx_); // Errors if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::open_array_mtx_init() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_init(&open_array_omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_init(&open_array_pthread_mtx_); // Errors if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::open_array_mtx_lock() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_lock(&open_array_omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_lock(&open_array_pthread_mtx_); // Errors if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::open_array_mtx_unlock() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_unlock(&open_array_omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_unlock(&open_array_pthread_mtx_); // Errors if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } void StorageManager::sort_fragment_names( std::vector& fragment_names) const { // Initializations int fragment_num = fragment_names.size(); std::string t_str; int64_t stripped_fragment_name_size, t; std::vector > t_pos_vec; t_pos_vec.resize(fragment_num); // Get the timestamp for each fragment for(int i=0; i(t, i); break; } } } // Sort the names based on the timestamps SORT(t_pos_vec.begin(), t_pos_vec.end()); std::vector fragment_names_sorted; fragment_names_sorted.resize(fragment_num); for(int i=0; i all_dirs = ::get_dirs(fs_, workspace_real); for(auto const& dir: all_dirs) { if(is_group(fs_, dir)) { group_delete(dir); } else if(is_metadata(fs_, dir)) { metadata_delete(dir); } else if(is_array(fs_, dir)){ array_delete(dir); } else { // Non TileDB related std::string errmsg = std::string("Cannot delete non TileDB related element '") + dir + "'"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } } // Success return TILEDB_SM_OK; } int StorageManager::workspace_delete( const std::string& workspace) { // Get real paths std::string workspace_real = real_dir(fs_, workspace); // Check if workspace exists if(!is_workspace(fs_, workspace_real)) { std::string errmsg = std::string("Workspace '") + workspace_real + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Clear workspace if(workspace_clear(workspace_real) != TILEDB_SM_OK) return TILEDB_SM_ERR; // Delete directory if(delete_dir(fs_, workspace_real) != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::workspace_move( const std::string& old_workspace, const std::string& new_workspace) { // Get real paths std::string old_workspace_real = real_dir(fs_, old_workspace); std::string new_workspace_real = real_dir(fs_, new_workspace); // Check if old workspace exists if(!is_workspace(fs_, old_workspace_real)) { std::string errmsg = std::string("Workspace '") + old_workspace_real + "' does not exist"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Check new workspace if(new_workspace_real == "") { std::string errmsg = std::string("Invalid workspace '") + new_workspace_real + "'"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } if(is_dir(fs_, new_workspace_real)) { std::string errmsg = std::string("Directory '") + new_workspace_real + "' already exists"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // New workspace should not be inside another workspace, group array or // metadata std::string new_workspace_real_parent = parent_dir(fs_, new_workspace_real); if(is_workspace(fs_, new_workspace_real_parent) || is_group(fs_, new_workspace_real_parent) || is_array(fs_, new_workspace_real_parent) || is_metadata(fs_, new_workspace_real_parent)) { std::string errmsg = std::string("Folder '") + new_workspace_real_parent + "' should not be a workspace, group, array, or metadata"; PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Rename directory if(move_path(fs_, old_workspace_real, new_workspace_real)) { std::string errmsg = std::string("Cannot move group; ") + strerror(errno); PRINT_ERROR(errmsg); tiledb_sm_errmsg = TILEDB_SM_ERRMSG + errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::OpenArray::mutex_destroy() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_destroy(&omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_destroy(&pthread_mtx_); // Error if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::OpenArray::mutex_init() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_init(&omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_init(&pthread_mtx_); // Error if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::OpenArray::mutex_lock() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_lock(&omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_lock(&pthread_mtx_); // Error if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } int StorageManager::OpenArray::mutex_unlock() { #ifdef HAVE_OPENMP int rc_omp_mtx = ::mutex_unlock(&omp_mtx_); #else int rc_omp_mtx = TILEDB_UT_OK; #endif int rc_pthread_mtx = ::mutex_unlock(&pthread_mtx_); // Error if(rc_pthread_mtx != TILEDB_UT_OK || rc_omp_mtx != TILEDB_UT_OK) { tiledb_sm_errmsg = tiledb_ut_errmsg; return TILEDB_SM_ERR; } // Success return TILEDB_SM_OK; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_manager_config.cc000066400000000000000000000141111453617025200265200ustar00rootroot00000000000000/** * @file storage_manager_config.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This file implements the StorageManagerConfig class. */ #include "storage_azure_blob.h" #include "storage_gcs.h" #include "storage_s3.h" #include "storage_manager_config.h" #include "tiledb_constants.h" #include "utils.h" #include #include #include #include /* ****************************** */ /* MACROS */ /* ****************************** */ #ifdef TILEDB_VERBOSE # define PRINT_ERROR(x) std::cerr << TILEDB_SMC_ERRMSG << x << ".\n" #else # define PRINT_ERROR(x) do { } while(0) #endif /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ std::string tiledb_smc_errmsg = ""; /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ StorageManagerConfig::StorageManagerConfig() { // Default values fs_ = new PosixFS(); home_ = ""; read_method_ = TILEDB_IO_MMAP; write_method_ = TILEDB_IO_WRITE; #ifdef HAVE_MPI mpi_comm_ = NULL; #endif } StorageManagerConfig::~StorageManagerConfig() { if (fs_ != NULL) { delete fs_; } } /* ****************************** */ /* MUTATORS */ /* ****************************** */ int StorageManagerConfig::init( const char* home, #ifdef HAVE_MPI MPI_Comm* mpi_comm, #endif int read_method, int write_method, const bool enable_shared_posixfs_optimizations) { // Initialize home if (home != NULL && strstr(home, "://")) { if (fs_ != NULL) { delete fs_; fs_ = NULL; } home_ = std::string(home, strlen(home)); std::string errmsg; if (is_azure_blob_storage_path(home_)) { try { fs_ = new AzureBlob(home_); } catch(std::system_error& ex) { errmsg = "Azure Storage Blob initialization failed for home=" + home_ + "; " + tiledb_fs_errmsg + "; " + ex.what(); PRINT_ERROR(errmsg); tiledb_smc_errmsg = TILEDB_SMC_ERRMSG + errmsg; return TILEDB_SMC_ERR; } } else if (is_s3_storage_path(home_)) { try { fs_ = new S3(home_); } catch(std::system_error& ex) { errmsg = "S3 Storage initialization failed for home=" + home_ + "; " + tiledb_fs_errmsg + "; " + ex.what(); PRINT_ERROR(ex.what()); tiledb_smc_errmsg = TILEDB_SMC_ERRMSG + errmsg; return TILEDB_SMC_ERR; } } else if (is_gcs_path(home_)) { try { fs_ = new GCS(home_); } catch(std::system_error& ex) { errmsg = "GCS Storage initialization failed for home=" + home_ + "; " + tiledb_fs_errmsg + "; " + ex.what(); PRINT_ERROR(ex.what()); tiledb_smc_errmsg = TILEDB_SMC_ERRMSG + errmsg; return TILEDB_SMC_ERR; } } else if (is_supported_cloud_path(home_)) { try { #ifdef USE_HDFS fs_ = new HDFS(home_); #else throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "TileDB built with HDFS support disabled."); #endif } catch(std::system_error& ex) { errmsg = "HDFS initialization failed for home=" + home_ + "; " + tiledb_fs_errmsg + "; " + ex.what(); PRINT_ERROR(ex.what()); tiledb_smc_errmsg = TILEDB_SMC_ERRMSG + errmsg; return TILEDB_SMC_ERR; } } else { tiledb_smc_errmsg = "No TileDB support for home=" + home_; PRINT_ERROR(tiledb_smc_errmsg); return TILEDB_SMC_ERR; } read_method_ = TILEDB_IO_READ; write_method_ = TILEDB_IO_WRITE; return TILEDB_SMC_OK; } // Default Posix case assert(fs_ != NULL); dynamic_cast(fs_)->set_disable_file_locking(enable_shared_posixfs_optimizations); dynamic_cast(fs_)->set_keep_write_file_handles_open(enable_shared_posixfs_optimizations); if(home == NULL) { home_ = ""; } else { home_ = std::string(home, strlen(home)); } #ifdef HAVE_MPI // Initialize MPI communicator mpi_comm_ = mpi_comm; #endif // Initialize read method read_method_ = read_method; if(read_method_ != TILEDB_IO_READ && read_method_ != TILEDB_IO_MMAP && read_method_ != TILEDB_IO_MPI) read_method_ = TILEDB_IO_MMAP; // Use default // Initialize write method write_method_ = write_method; if(write_method_ != TILEDB_IO_WRITE && write_method_ != TILEDB_IO_MPI) write_method_ = TILEDB_IO_WRITE; // Use default return TILEDB_SMC_OK; } /* ****************************** */ /* ACCESSORS */ /* ****************************** */ const std::string& StorageManagerConfig::home() const { return home_; } #ifdef HAVE_MPI MPI_Comm* StorageManagerConfig::mpi_comm() const { return mpi_comm_; } #endif int StorageManagerConfig::read_method() const { return read_method_; } int StorageManagerConfig::write_method() const { return write_method_; } StorageFS* StorageManagerConfig::get_filesystem() const { return fs_; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_posixfs.cc000066400000000000000000000371311453617025200252630ustar00rootroot00000000000000/** * @file local_fs.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Default Posix Filesystem Implementation for StorageFS */ #include "error.h" #include "storage_posixfs.h" #include "utils.h" #include #include #include #include #include #include #include #include #include #include #define POSIX_ERROR(MSG, PATH) SYSTEM_ERROR(TILEDB_FS_ERRMSG, MSG, PATH, tiledb_fs_errmsg) static int sync_kernel(int fd, bool locking_support, std::string filename); PosixFS::~PosixFS() { for (auto it = write_map_.begin(); it != write_map_.end(); ++it) { std::string filename = it->first; int fd = it->second; POSIX_ERROR("File does not seem to be closed", filename); sync_kernel(fd, true, filename); if (close(fd)) { POSIX_ERROR("Could not close file from destructor", filename); } } write_map_.clear(); } std::string PosixFS::current_dir() { std::string dir = ""; char* path = getcwd(NULL, 0); if(path != NULL) { dir = path; free(path); } return dir; } int PosixFS::set_working_dir(const std::string& dir) { reset_errno(); if (chdir(dir.c_str())) { POSIX_ERROR("Cannot set working dir", dir); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } bool PosixFS::is_dir(const std::string& dir) { struct stat st; memset(&st, 0, sizeof(struct stat)); return !stat(dir.c_str(), &st) && S_ISDIR(st.st_mode); } bool PosixFS::is_file(const std::string& file) { struct stat st; memset(&st, 0, sizeof(struct stat)); return !stat(file.c_str(), &st) && S_ISREG(st.st_mode); } void adjacent_slashes_dedup(std::string& value) { value.erase(std::unique(value.begin(), value.end(), both_slashes), value.end()); } void purge_dots_from_path(std::string& path) { // For easy reference size_t path_size = path.size(); // Trivial case if(path_size == 0 || path == "/") return; // It expects an absolute path assert(path[0] == '/'); // Tokenize const char* token_c_str = path.c_str() + 1; std::vector tokens, final_tokens; std::string token; for(size_t i=1; ireal_dir(dir); // If the directory does not exist, create it if(!is_dir(real_dir)) { if(mkdir(real_dir.c_str(), S_IRWXU)) { POSIX_ERROR("Cannot create directory", real_dir); return TILEDB_FS_ERR; } } else { // Error POSIX_ERROR("Cannot create directory; Directory already exists", real_dir); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } static int delete_file_nftw_cb(const char *filepath, const struct stat *ptr, int flag, struct FTW *ftwbuf) { if (remove(filepath)) { POSIX_ERROR("Could not remove file", filepath); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int PosixFS::delete_dir(const std::string& dirname) { reset_errno(); // Get real path std::string dirname_real = this->real_dir(dirname); if (nftw(dirname_real.c_str(), delete_file_nftw_cb, 64, FTW_DEPTH | FTW_PHYS)) { POSIX_ERROR("Could not recursively delete directory", dirname); return TILEDB_FS_ERR; } // Success return TILEDB_FS_OK; } std::vector PosixFS::get_dirs(const std::string& dir) { reset_errno(); std::vector dirs; std::string new_dir; struct dirent *next_file; DIR* c_dir = opendir(dir.c_str()); if(c_dir == NULL) { POSIX_ERROR("Cannot open directory", dir); return std::vector(); } while((next_file = readdir(c_dir))) { if(!strcmp(next_file->d_name, ".") || !strcmp(next_file->d_name, "..") || !is_dir(dir + "/" + next_file->d_name)) continue; new_dir = dir + "/" + next_file->d_name; dirs.push_back(new_dir); } // Close array directory if (closedir(c_dir)) { POSIX_ERROR("Cannot close directory", dir); } // Return return dirs; } std::vector PosixFS::get_files(const std::string& dir) { reset_errno(); std::vector files; std::string filename; struct dirent *next_file; DIR* c_dir = opendir(dir.c_str()); if(c_dir == NULL) { POSIX_ERROR("Cannot open directory", dir); return std::vector(); } while((next_file = readdir(c_dir))) { if(!strcmp(next_file->d_name, ".") || !strcmp(next_file->d_name, "..") || !is_file(dir + "/" + next_file->d_name)) continue; filename = dir + "/" + next_file->d_name; files.push_back(filename); } // Close array directory if (closedir(c_dir)) { POSIX_ERROR("Cannot close directory", dir); } // Return return files; } int PosixFS::create_file(const std::string& filename, int flags, mode_t mode) { reset_errno(); int fd = open(filename.c_str(), flags, mode); if(fd == -1 || close(fd)) { POSIX_ERROR("Failed to create file", filename); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int PosixFS::delete_file(const std::string& filename) { reset_errno(); if(remove(filename.c_str())) { POSIX_ERROR("Cannot remove file", filename); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } ssize_t PosixFS::file_size(const std::string& filename) { reset_errno(); if (!is_file(filename)) { return TILEDB_FS_ERR; } int fd = open(filename.c_str(), O_RDONLY); if(fd == -1) { POSIX_ERROR("Cannot get file size; File opening error", filename); return TILEDB_FS_ERR; } struct stat st; memset(&st, 0, sizeof(struct stat)); fstat(fd, &st); off_t file_size = st.st_size; if (close(fd)) { POSIX_ERROR("Cannot get file size; File closing error", filename); } return file_size; } static int get_fd(const std::string& filename, std::unordered_map& map, std::mutex& mtx) { std::lock_guard lock(mtx); auto search = map.find(filename); if (search != map.end()) { return search->second; } else { return -1; } } static void set_fd(const std::string& filename, int fd, std::unordered_map& map, std::mutex& mtx) { std::unique_lock lock(mtx); map.emplace(filename, fd); } static void remove_fd(const std::string& filename, std::unordered_map& map, std::mutex& mtx) { std::unique_lock lock(mtx); map.erase(filename); } int PosixFS::read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length) { reset_errno(); if (length == 0) { return TILEDB_FS_OK; } // Not supporting simultaneous read/writes. if (keep_write_file_handles_open() && get_fd(filename, write_map_, write_map_mtx_) >= 0) { POSIX_ERROR("Cannot open simultaneously for reads/writes", filename); return TILEDB_FS_ERR; } // Open file int fd = open(filename.c_str(), O_RDONLY); if (fd == -1) { POSIX_ERROR("Cannot read from file; File opening error", filename); return TILEDB_FS_ERR; } // Read in batches of TILEDB_UT_MAX_WRITE_COUNT size_t nbytes = 0; char *pbuf = reinterpret_cast(buffer); int rc = TILEDB_FS_OK; do { ssize_t bytes_read = pread(fd, reinterpret_cast(pbuf), (length - nbytes) > TILEDB_UT_MAX_WRITE_COUNT?TILEDB_UT_MAX_WRITE_COUNT : length-nbytes, offset + nbytes); if (bytes_read < 0) { POSIX_ERROR("Cannot read from file; File reading error", filename); rc = TILEDB_FS_ERR; } else if (bytes_read == 0) { POSIX_ERROR("EOF reached; File reading error", filename); rc = TILEDB_FS_ERR; } else { nbytes += bytes_read; pbuf += bytes_read; } } while (nbytes < length && rc == TILEDB_FS_OK); // Close file if (close(fd)) { POSIX_ERROR("Cannot read from file; File closing error", filename); return TILEDB_FS_ERR; } return rc; } static int write_to_file_kernel(int fd, const void *buffer, size_t buffer_size) { // Write in batches of TILEDB_UT_MAX_WRITE_COUNT size_t nbytes = 0; char *pbuf = reinterpret_cast(const_cast(buffer)); do { size_t count = (buffer_size - nbytes) > TILEDB_UT_MAX_WRITE_COUNT ? TILEDB_UT_MAX_WRITE_COUNT : buffer_size - nbytes; assert(count != 0); ssize_t bytes_written = write(fd, reinterpret_cast(pbuf), count); if(bytes_written < 0) { return TILEDB_FS_ERR; } nbytes += bytes_written; pbuf += bytes_written; } while (nbytes < buffer_size); return TILEDB_FS_OK; } int PosixFS::write_to_file_keep_file_handles_open(const std::string& filename, const void *buffer, size_t buffer_size) { int fd = get_fd(filename, write_map_, write_map_mtx_); if (fd == -1) { // Open file fd = open(filename.c_str(), O_WRONLY | O_APPEND | O_CREAT, S_IRWXU); if(fd == -1) { POSIX_ERROR("Cannot write to file; File opening error", filename); return TILEDB_FS_ERR; } set_fd(filename, fd, write_map_, write_map_mtx_); } if (write_to_file_kernel(fd, buffer, buffer_size)) { POSIX_ERROR("Cannot write to file; File writing error", filename); close(fd); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int PosixFS::write_to_file(const std::string& filename, const void *buffer, size_t buffer_size) { reset_errno(); if (buffer_size == 0) { return TILEDB_FS_OK; } if (keep_write_file_handles_open()) { return write_to_file_keep_file_handles_open(filename, buffer, buffer_size); } // Open file int fd = open(filename.c_str(), O_WRONLY | O_APPEND | O_CREAT, S_IRWXU); if(fd == -1) { POSIX_ERROR("Cannot write to file; File opening error", filename); return TILEDB_FS_ERR; } if (write_to_file_kernel(fd, buffer, buffer_size)) { POSIX_ERROR("Cannot write to file; File writing error", filename); close(fd); return TILEDB_FS_ERR; } // Close file if (close(fd)) { POSIX_ERROR("Cannot write to file; File closing error", filename); return TILEDB_FS_ERR; } // Success return TILEDB_FS_OK; } int PosixFS::move_path(const std::string& old_path, const std::string& new_path) { reset_errno(); if(rename(old_path.c_str(), new_path.c_str())) { POSIX_ERROR("Cannot rename path", old_path); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } static int sync_kernel(int fd, bool locking_support, std::string filename) { // Sync if(fsync(fd)) { // Ignoring EINVAL errors on fsync that can show up on NFS/CIFS even if they are posix compilant" if (errno != EINVAL && locking_support) { POSIX_ERROR("Cannot sync file; File syncing error. Some network filesystems(NFS/CIFS) can have issues with fsync due to synchronization across machines. Try setting env \"export TILEDB_DISABLE_FILE_LOCKING=1\" and retry", filename); return TILEDB_FS_ERR; } } return TILEDB_FS_OK; } int PosixFS::sync_path(const std::string& filename) { reset_errno(); int fd = get_fd(filename, write_map_, write_map_mtx_); if (fd != -1) { return sync_kernel(fd, locking_support(), filename); } // Open file if(is_dir(filename)) { // DIRECTORY fd = open(filename.c_str(), O_RDONLY, S_IRWXU); } else if(is_file(filename)) { // FILE fd = open(filename.c_str(), O_WRONLY | O_APPEND, S_IRWXU); } else { return TILEDB_FS_OK; // If file does not exist, exit } // Handle error if(fd == -1) { POSIX_ERROR("Cannot sync file; File opening error", filename); return TILEDB_FS_ERR; } // Sync sync_kernel(fd, locking_support(), filename); // Close file if(close(fd)) { POSIX_ERROR("Cannot sync file; File closing error", filename); return TILEDB_FS_ERR; } // Success return TILEDB_FS_OK; } int PosixFS::close_file(const std::string& filename) { if (keep_write_file_handles_open()) { int fd = get_fd(filename, write_map_, write_map_mtx_); if (fd >= 0) { int rc = close(fd); remove_fd(filename, write_map_, write_map_mtx_); if (rc) { POSIX_ERROR("Cannot close file; File closing error", filename); return TILEDB_FS_ERR; } } } return TILEDB_FS_OK; } bool PosixFS::locking_support() { return !disable_file_locking(); } void PosixFS::set_keep_write_file_handles_open(const bool val) { keep_write_file_handles_open_ = val; } bool PosixFS::keep_write_file_handles_open() { if (keep_write_file_handles_open_set_) { return keep_write_file_handles_open_; } if (getenv("TILEDB_KEEP_FILE_HANDLES_OPEN")) { keep_write_file_handles_open_ = is_env_set("TILEDB_KEEP_FILE_HANDLES_OPEN"); } keep_write_file_handles_open_set_ = true; return keep_write_file_handles_open_; } void PosixFS::set_disable_file_locking(const bool val) { disable_file_locking_ = val; } bool PosixFS::disable_file_locking() { if (is_disable_file_locking_set) { return disable_file_locking_; } if (getenv("TILEDB_DISABLE_FILE_LOCKING")) { disable_file_locking_ = is_env_set("TILEDB_DISABLE_FILE_LOCKING"); } is_disable_file_locking_set = true; return disable_file_locking_; } genomicsdb-0.0~git20231212.9d7ddd0/core/src/storage/storage_s3.cc000066400000000000000000000441321453617025200241140ustar00rootroot00000000000000/** * @file storage_s3.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * AWS S3 Support for StorageFS */ #include "error.h" #include "storage_s3.h" #include "uri.h" #include "utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include // errno is not relevant with aws sdk, so setting to 0 #define S3_ERROR(MSG, PATH) PATH_ERROR(TILEDB_FS_ERRMSG, "S3: "+MSG, PATH, tiledb_fs_errmsg) #define S3_ERROR1(MSG, OUTCOME, PATH) PATH_ERROR(TILEDB_FS_ERRMSG, "S3: "+MSG+" "+OUTCOME.GetError().GetExceptionName()+" "+OUTCOME.GetError().GetMessage(), PATH, tiledb_fs_errmsg) #define CLASS_TAG "TILEDB_STORAGE_S3" // ugly, as we cannot call Aws::ShutdownAPI(memory leak??) but no other option for now - // See https://github.com/aws/aws-sdk-cpp/issues/456 and // https://github.com/aws/aws-sdk-cpp/issues/1067 std::once_flag awssdk_init_api_flag; std::shared_ptr awssdk_options; void awssdk_init_api() { awssdk_options = std::make_shared(); // awssdk_options->loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace; Aws::InitAPI(*awssdk_options.get()); } S3::S3(const std::string& home) { s3_uri path_uri(home); if (path_uri.protocol().compare("s3") != 0) { throw std::system_error(EPROTONOSUPPORT, std::generic_category(), "S3 FS only supports s3:// URI protocols"); } if (path_uri.bucket().size() == 0) { throw std::system_error(EPROTO, std::generic_category(), "S3 URI does not seem to have a bucket specified"); } std::call_once(awssdk_init_api_flag, awssdk_init_api); Aws::Client::ClientConfiguration client_config; client_config.scheme = Aws::Http::Scheme::HTTPS; // SSL/TLS only for HIPPA/PHI Compliance // Check for endpoint_override. This environment variable is not exposed by aws sdk and is specific to TileDB. auto env_var = getenv("AWS_ENDPOINT_OVERRIDE"); bool useVirtualAddressing = true; // default if (env_var) { client_config.endpointOverride = Aws::String(env_var); useVirtualAddressing = false; } std::shared_ptr retryStrategy = std::make_shared(15, 2); client_config.retryStrategy = retryStrategy; std::string ca_certs_location = locate_ca_certs(); if (!ca_certs_location.empty()) { client_config.caFile = ca_certs_location.c_str(); } #if(0) // Used with valgrind. Probably need to make this configurable client_config.requestTimeoutMs=30000; client_config.connectTimeoutMs=30000; #endif client_ = std::make_shared(client_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, useVirtualAddressing); Aws::S3::Model::HeadBucketRequest head_request; head_request.WithBucket(path_uri.bucket()); auto outcome = client_->HeadBucket(head_request); if (!outcome.IsSuccess()) { S3_ERROR1("Failed to locate bucket", outcome, home); throw std::system_error(EIO, std::generic_category(), "S3 FS only supports already existing buckets. Create bucket from either the aws CLI or the aws storage portal before restarting operation"); } bucket_name_ = path_uri.bucket(); working_dir_ = get_path(path_uri.path()); // Set default buffer sizes, overridden with env vars TILEDB_DOWNLOAD_BUFFER_SIZE and TILEDB_UPLOAD_BUFFER_SIZE // Minimum size for each part of a multipart upload, except for the last part, using the same value for both uploads and downloads for now // see https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html download_buffer_size_ = 5*1024*1024; // 5M upload_buffer_size_ = 5*1024*1024; // 5M } S3::~S3() { // Check for all the files that have not been committed std::vector uncommitted_files; for (auto it=write_map_.begin(); it!=write_map_.end(); ++it) { uncommitted_files.push_back(it->first); } for (auto filename : uncommitted_files) { commit_file(filename); } } std::string S3::current_dir() { return working_dir_; } int S3::set_working_dir(const std::string& dir) { working_dir_ = get_path(dir); return TILEDB_FS_OK; } bool S3::path_exists(const std::string& path) { auto aws_path = to_aws_string(get_path(path)); Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket_name_); request.SetKey(aws_path); if (client_->HeadObject(request).IsSuccess()) { return true; } else if (path.back() == '/') { Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket_name_); request.SetPrefix(aws_path); request.SetDelimiter(to_aws_string(DELIMITER)); request.SetMaxKeys(1); auto outcome = client_->ListObjectsV2(request); if (outcome.IsSuccess()) { return outcome.GetResult().GetContents().size() || outcome.GetResult().GetCommonPrefixes().size(); } } return false; } std::string S3::real_dir(const std::string& dir) { if (dir.find("://") != std::string::npos) { s3_uri path_uri(dir); if (path_uri.bucket().compare(bucket_name_)) { throw std::runtime_error("Credentialed account during instantiation does not match the uri passed to real_dir. Aborting"); } } return get_path(dir); } int S3::create_path(const std::string& path) { Aws::S3::Model::PutObjectRequest request; request.SetBucket(bucket_name_); request.SetKey(to_aws_string(get_path(path))); auto outcome = client_->PutObject(request); if (!outcome.IsSuccess()) { S3_ERROR1("Could not create path", outcome, path); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int S3::create_dir(const std::string& dir) { if (is_dir(dir) || is_file(dir)) { S3_ERROR("Path already exists", dir); return TILEDB_FS_ERR; } /* if (working_dir_.compare(dir) && !is_dir(parent_dir(NULL, dir))) { S3_ERROR("Parent directory to path does not exist", dir); return TILEDB_FS_ERR; } */ return create_path(slashify(dir)); } int S3::delete_path(const std::string& path) { auto aws_path = to_aws_string(get_path(path)); Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(bucket_name_); request.SetKey(aws_path); auto outcome = client_->DeleteObject(request); if (!outcome.IsSuccess()) { S3_ERROR1("Could not delete path", outcome, path); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int S3::delete_dir(const std::string& dir) { if (is_file(dir)) { S3_ERROR("Cannot delete dir as it seems to be a file", dir); return TILEDB_FS_ERR; } if (!is_dir(dir)) { S3_ERROR("Cannot delete non-existent dir", dir); return TILEDB_FS_ERR; } auto path = to_aws_string(slashify(get_path(dir))); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket_name_); request.SetPrefix(path); request.SetDelimiter(to_aws_string(DELIMITER)); Aws::String continuation_token = ""; int rc = TILEDB_FS_OK; do { if (continuation_token.length() > 0) { request.SetContinuationToken(continuation_token); } auto outcome = client_->ListObjectsV2(request); if (!outcome.IsSuccess()) { rc = TILEDB_FS_ERR; break; } else { continuation_token = outcome.GetResult().GetNextContinuationToken(); // Delete all files in dirs for (const auto& object : outcome.GetResult().GetContents()) { if (is_file(object.GetKey())) { rc |= delete_path(object.GetKey()); } } // Recursively delete children dirs // TODO: Remove recursion for performance for (const auto& object : outcome.GetResult().GetCommonPrefixes()) { rc |= delete_dir(object.GetPrefix()); } } } while (continuation_token.length() > 0); // Check if dir exists again as some object stores(e.g. minio) remove the // directory after removing all content if (is_dir(dir)) { rc |= delete_path(slashify(dir)); } return rc; } std::vector S3::get_dirs(const std::string& dir) { std::vector dirs; auto path = to_aws_string(slashify(get_path(dir))); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket_name_); request.SetPrefix(path); request.SetDelimiter(to_aws_string(DELIMITER)); Aws::String continuation_token = ""; do { if (continuation_token.length() > 0) { request.SetContinuationToken(continuation_token); } auto outcome = client_->ListObjectsV2(request); if (outcome.IsSuccess()) { continuation_token = outcome.GetResult().GetNextContinuationToken(); for (const auto& object : outcome.GetResult().GetCommonPrefixes()) { dirs.push_back(unslashify(object.GetPrefix())); } } } while (continuation_token.length() > 0); return dirs; } std::vector S3::get_files(const std::string& dir) { std::vector files; auto path = to_aws_string(slashify(get_path(dir))); Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucket_name_); request.SetPrefix(path); request.SetDelimiter(to_aws_string(DELIMITER)); Aws::String continuation_token = ""; do { if (continuation_token.length() > 0) { request.SetContinuationToken(continuation_token); } auto outcome = client_->ListObjectsV2(request); if (outcome.IsSuccess()) { continuation_token = outcome.GetResult().GetNextContinuationToken(); for (const auto& object : outcome.GetResult().GetContents()) { if (is_file(object.GetKey())) { files.push_back(object.GetKey()); } } } } while (continuation_token.length() > 0); return files; } int S3::create_file(const std::string& filename, int flags, mode_t mode) { if (is_dir(filename) || is_file(filename)) { S3_ERROR("Cannot create path as it already exists", filename); return TILEDB_FS_ERR; } return create_path(filename); } int S3::delete_file(const std::string& filename) { if (!is_file(filename)) { S3_ERROR("Cannot delete non-existent or non-file path", filename); return TILEDB_FS_ERR; } return delete_path(filename); } ssize_t S3::file_size(const std::string& filename) { auto aws_path = to_aws_string(get_path(filename)); Aws::S3::Model::HeadObjectRequest request; request.SetBucket(bucket_name_); request.SetKey(aws_path); auto outcome = client_->HeadObject(request); if (outcome.IsSuccess()) { return outcome.GetResult().GetContentLength(); } return TILEDB_FS_ERR; } class PreallocatedIOStream : public Aws::IOStream { public: PreallocatedIOStream(unsigned char* buffer, size_t length) : Aws::IOStream( Aws::New(CLASS_TAG, buffer, length)) { } ~PreallocatedIOStream() { Aws::Delete(rdbuf()); } }; int S3::read_from_file(const std::string& filename, off_t offset, void *buffer, size_t length) { if (length == 0) { return TILEDB_FS_OK; // Nothing to read } Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket_name_); request.SetKey(to_aws_string(get_path(filename))); request.SetRange(to_aws_string("bytes=" + std::to_string(offset) + "-" + std::to_string(offset+length-1))); request.SetResponseStreamFactory([buffer, length]() { unsigned char* buf = reinterpret_cast(const_cast(buffer)); return Aws::New(CLASS_TAG, buf, length); }); auto outcome = client_->GetObject(request); if (!outcome.IsSuccess()) { S3_ERROR1("Failed to get object", outcome, filename); return TILEDB_FS_ERR; } auto content_length = outcome.GetResult().GetContentLength(); if (content_length < 0 || (size_t)content_length < length) { S3_ERROR("Could not read the file for bytes of length=" + std::to_string(length) + " from offset=" + std::to_string(offset), filename); return TILEDB_FS_ERR; } return TILEDB_FS_OK; } int S3::write_to_file(const std::string& filename, const void *buffer, size_t buffer_size) { if (buffer_size == 0) { return create_file(filename, 0, 0); } // Initialize upload std::string filepath = get_path(filename); auto aws_filename = to_aws_string(filepath); std::string upload_id; size_t part_number = 0; std::shared_ptr completed_parts; { const std::lock_guard lock(write_map_mtx_); auto search = write_map_.find(filepath); if (search == write_map_.end()) { // Filepath not in write_map, so start by initiating the request Aws::S3::Model::CreateMultipartUploadRequest create_request; create_request.SetBucket(bucket_name_); create_request.SetKey(aws_filename); auto createMultipartUploadOutcome = client_->CreateMultipartUpload(create_request); std::string upload_id = createMultipartUploadOutcome.GetResult().GetUploadId(); auto update_info = multipart_upload_info_t(upload_id); write_map_.insert({filepath, std::move(update_info)}); } auto found = write_map_.find(filepath); assert(found != write_map_.end()); found->second.part_number_++; upload_id = found->second.upload_id_; part_number = found->second.part_number_; completed_parts = found->second.completed_parts_; // Verify that the previous uploaded part was at least 5M - see https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html // S3 throws error only when committing, so it is better to check in write_to_file and abort here. Note that cleanup occurs // in commit_file that is called from the destructor if there was an issue. auto last_uploaded_size = found->second.last_uploaded_size_; if (found->second.abort_upload_ || (last_uploaded_size != 0 && last_uploaded_size < 5*1024*1024)) { S3_ERROR("Only the last of the uploadable parts can be less than 5MB", filepath); found->second.abort_upload_ = true; return TILEDB_FS_ERR; } else { found->second.last_uploaded_size_ = buffer_size; } } assert(!upload_id.empty()); assert(part_number > 0); // Start upload Aws::S3::Model::UploadPartRequest request; request.SetBucket(bucket_name_); request.SetKey(aws_filename); request.SetPartNumber(part_number); request.SetUploadId(to_aws_string(upload_id)); unsigned char* buf = reinterpret_cast(const_cast(buffer)); request.SetBody(Aws::MakeShared(CLASS_TAG, buf, buffer_size)); request.SetContentLength(buffer_size); auto outcome = client_->UploadPartCallable(request); auto etag = outcome.get().GetResult().GetETag(); if (etag.empty()) { S3_ERROR("UploadPartCallable not successful as etag is empty", filename); return TILEDB_FS_ERR; } // Add to list of uploaded parts Aws::S3::Model::CompletedPart completed_part; completed_part.SetPartNumber(part_number); completed_part.SetETag(etag); completed_parts->AddParts(std::move(completed_part)); return TILEDB_FS_OK; } int S3::commit_file(const std::string& filename) { std::string filepath = get_path(filename); auto aws_filename = to_aws_string(filepath); std::string upload_id; std::shared_ptr completed_parts; bool abort_upload = false; int rc = TILEDB_FS_OK; { const std::lock_guard lock(write_map_mtx_); auto found = write_map_.find(filepath); if (found != write_map_.end()) { auto multipart_update_info = found->second; upload_id = multipart_update_info.upload_id_; completed_parts = multipart_update_info.completed_parts_; abort_upload = multipart_update_info.abort_upload_; write_map_.erase(filepath); } } if (!upload_id.empty()) { if (!abort_upload) { Aws::S3::Model::CompleteMultipartUploadRequest finish_upload_request; finish_upload_request.SetBucket(bucket_name_); finish_upload_request.SetKey(aws_filename); finish_upload_request.SetUploadId(to_aws_string(upload_id)); finish_upload_request.WithMultipartUpload(*completed_parts); auto finish_upload_outcome = client_->CompleteMultipartUpload(finish_upload_request); if (!finish_upload_outcome.IsSuccess()) { std::string msg = "Could not upload successfully for upload_id=" + upload_id; S3_ERROR1(msg, finish_upload_outcome, filename); abort_upload = true; rc = TILEDB_FS_ERR; } } if (abort_upload) { Aws::S3::Model::AbortMultipartUploadRequest abort_upload_request; abort_upload_request.SetBucket(bucket_name_); abort_upload_request.SetKey(aws_filename); abort_upload_request.SetUploadId(to_aws_string(upload_id)); auto abort_upload_outcome = client_->AbortMultipartUpload(abort_upload_request); if (!abort_upload_outcome.IsSuccess()) { S3_ERROR1("Could not abort upload successfullt", abort_upload_outcome, filename); rc = TILEDB_FS_ERR; } } } return rc; } genomicsdb-0.0~git20231212.9d7ddd0/deps/000077500000000000000000000000001453617025200173005ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/deps/HDFSWrapper/000077500000000000000000000000001453617025200213655ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/deps/azure-storage-cpplite/000077500000000000000000000000001453617025200235265ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/deps/muparserx/000077500000000000000000000000001453617025200213265ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/doc/000077500000000000000000000000001453617025200171125ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/doc/Doxyfile.mk000066400000000000000000003066671453617025200212500ustar00rootroot00000000000000# Doxyfile 1.8.8 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. # # All text after a double hash (##) is considered a comment and is placed in # front of the TAG it is preceding. # # All text after a single hash (#) is considered a comment and will be ignored. # The format is: # TAG = value [value, ...] # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file # that follow. The default is UTF-8 which is also the encoding used for all text # before the first occurrence of this tag. Doxygen uses libiconv (or the iconv # built into libc) for the transcoding. See http://www.gnu.org/software/libiconv # for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by # double-quotes, unless you are using Doxywizard) that should identify the # project for which the documentation is generated. This name is used in the # title of most generated pages and in a few other places. # The default value is: My Project. PROJECT_NAME = "TileDB" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. PROJECT_NUMBER = # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = # With the PROJECT_LOGO tag one can specify an logo or icon that is included in # the documentation. The maximum height of the logo should not exceed 55 pixels # and the maximum width should not exceed 200 pixels. Doxygen will copy the logo # to the output directory. PROJECT_LOGO = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. OUTPUT_DIRECTORY = # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and # will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes # performance problems for the file system. # The default value is: NO. CREATE_SUBDIRS = NO # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode # U+3044. # The default value is: NO. ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, # Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), # Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, # Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), # Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, # Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, # Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, # Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. # The default value is: YES. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief # description of a member or function before the detailed description # # Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. # The default value is: YES. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator that is # used to form the text in various listings. Each string in this list, if found # as the leading text of the brief description, will be stripped from the text # and the result, after processing the whole list, is used as the annotated # text. Otherwise, the brief description is used as-is. If left blank, the # following values are used ($name is automatically replaced with the name of # the entity):The $name class, The $name widget, The $name file, is, provides, # specifies, contains, represents, a, an and the. ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # doxygen will generate a detailed section even if there is only a brief # description. # The default value is: NO. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. # The default value is: NO. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the # shortest path that makes the file name unique will be used # The default value is: YES. FULL_PATH_NAMES = YES # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand # part of the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the path to # strip. # # Note that you can specify absolute paths here, but also relative paths, which # will be relative from the directory where doxygen is started. # This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the # path mentioned in the documentation of a class, which tells the reader which # header file to include in order to use a class. If left blank only the name of # the header file containing the class definition is used. Otherwise one should # specify the list of include paths that are normally passed to the compiler # using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't # support long names like on DOS, Mac, or CD-ROM. # The default value is: NO. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the # first line (until the first dot) of a Javadoc-style comment as the brief # description. If set to NO, the Javadoc-style will behave just like regular Qt- # style comments (thus requiring an explicit @brief command for a brief # description.) # The default value is: NO. JAVADOC_AUTOBRIEF = NO # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus # requiring an explicit \brief command for a brief description.) # The default value is: NO. QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a # multi-line C++ special comment block (i.e. a block of //! or /// comments) as # a brief description. This used to be the default behavior. The new default is # to treat a multi-line C++ comment block as a detailed description. Set this # tag to YES if you prefer the old behavior instead. # # Note that setting this tag to YES also means that rational rose comments are # not recognized any more. # The default value is: NO. MULTILINE_CPP_IS_BRIEF = NO # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a # new page for each member. If set to NO, the documentation of a member will be # part of the file/class/namespace that contains it. # The default value is: NO. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen # uses this value to replace tabs by spaces in code fragments. # Minimum value: 1, maximum value: 16, default value: 4. TAB_SIZE = 4 # This tag can be used to specify a number of aliases that act as commands in # the documentation. An alias has the form: # name=value # For example adding # "sideeffect=@par Side Effects:\n" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading # "Side Effects:". You can put \n's in the value part of an alias to insert # newlines. ALIASES = # This tag can be used to specify a number of word-keyword mappings (TCL only). # A mapping has the form "name=value". For example adding "class=itcl::class" # will allow you to use the command class in the itcl::class meaning. TCL_SUBST = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all # members will be omitted, etc. # The default value is: NO. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or # Python sources only. Doxygen will then generate output that is more tailored # for that language. For instance, namespaces will be presented as packages, # qualified scopes will look different, etc. # The default value is: NO. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources. Doxygen will then generate output that is tailored for Fortran. # The default value is: NO. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for VHDL. # The default value is: NO. OPTIMIZE_OUTPUT_VHDL = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, Javascript, # C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: # FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: # Fortran. In the later case the parser tries to guess whether the code is fixed # or free formatted code, this is the default for Fortran type files), VHDL. For # instance to make doxygen treat .inc files as Fortran files (default is PHP), # and .f files as C (default is Fortran), use: inc=Fortran f=C. # # Note For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by doxygen. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable # documentation. See http://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. # The default value is: YES. MARKDOWN_SUPPORT = YES # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by by putting a % sign in front of the word # or globally by setting AUTOLINK_SUPPORT to NO. # The default value is: YES. AUTOLINK_SUPPORT = YES # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this # tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); # versus func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. # The default value is: NO. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: # http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate # getter and setter methods for a property. Setting this option to YES will make # doxygen to replace the get and set methods by a property in the documentation. # This will only work if the methods are indeed getting or setting a simple # type. If this is not the case, or you want to show the methods anyway, you # should set this option to NO. # The default value is: YES. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. # The default value is: NO. DISTRIBUTE_GROUP_DOC = YES # Set the SUBGROUPING tag to YES to allow class member groups of the same type # (for instance a group of public functions) to be put as a subgroup of that # type (e.g. under the Public Functions section). Set it to NO to prevent # subgrouping. Alternatively, this can be done per class using the # \nosubgrouping command. # The default value is: YES. SUBGROUPING = YES # When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions # are shown inside the group in which they are included (e.g. using \ingroup) # instead of on a separate page (for HTML and Man pages) or section (for LaTeX # and RTF). # # Note that this feature does not work in combination with # SEPARATE_MEMBER_PAGES. # The default value is: NO. INLINE_GROUPED_CLASSES = NO # When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions # with only public data fields or simple typedef fields will be shown inline in # the documentation of the scope in which they are defined (i.e. file, # namespace, or group documentation), provided this scope is documented. If set # to NO, structs, classes, and unions are shown on a separate page (for HTML and # Man pages) or section (for LaTeX and RTF). # The default value is: NO. INLINE_SIMPLE_STRUCTS = NO # When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or # enum is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically be # useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. TYPEDEF_HIDES_STRUCT = NO # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be # an expensive process and often the same symbol appears multiple times in the # code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small # doxygen will become slower. If the cache is too large, memory is wasted. The # cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range # is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 # symbols. At the end of a run doxygen will report the cache usage and suggest # the optimal cache size from a speed point of view. # Minimum value: 0, maximum value: 9, default value: 0. LOOKUP_CACHE_SIZE = 0 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. Private # class members and static file members will be hidden unless the # EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. # Note: This will also disable the warnings about undocumented members that are # normally produced when WARNINGS is set to YES. # The default value is: NO. EXTRACT_ALL = NO # If the EXTRACT_PRIVATE tag is set to YES all private members of a class will # be included in the documentation. # The default value is: NO. EXTRACT_PRIVATE = YES # If the EXTRACT_PACKAGE tag is set to YES all members with package or internal # scope will be included in the documentation. # The default value is: NO. EXTRACT_PACKAGE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file will be # included in the documentation. # The default value is: NO. EXTRACT_STATIC = YES # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined # locally in source files will be included in the documentation. If set to NO # only classes defined in header files are included. Does not have any effect # for Java sources. # The default value is: YES. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local methods, # which are defined in the implementation section but not in the interface are # included in the documentation. If set to NO only methods in the interface are # included. # The default value is: NO. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base name of # the file that contains the anonymous namespace. By default anonymous namespace # are hidden. # The default value is: NO. EXTRACT_ANON_NSPACES = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation # section is generated. This option has no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO these classes will be included in the various overviews. This option has # no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend # (class|struct|union) declarations. If set to NO these declarations will be # included in the documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any # documentation blocks found inside the body of a function. If set to NO these # blocks will be appended to the function's detailed documentation block. # The default value is: NO. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation that is typed after a # \internal command is included. If the tag is set to NO then the documentation # will be excluded. Set it to YES to include the internal documentation. # The default value is: NO. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file # names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. # The default value is: system dependent. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES the # scope will be hidden. # The default value is: NO. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. SHOW_INCLUDE_FILES = YES # If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each # grouped member an include statement to the documentation, telling the reader # which file to include in order to use the member. # The default value is: NO. SHOW_GROUPED_MEMB_INC = NO # If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include # files with double quotes in the documentation rather than with sharp brackets. # The default value is: NO. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the # documentation for inline members. # The default value is: YES. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the # (detailed) documentation of file and class members alphabetically by member # name. If set to NO the members will appear in declaration order. # The default value is: YES. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief # descriptions of file, namespace and class members alphabetically by member # name. If set to NO the members will appear in declaration order. Note that # this will also influence the order of the classes in the class list. # The default value is: NO. SORT_BRIEF_DOCS = NO # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the # (brief and detailed) documentation of class members so that constructors and # destructors are listed first. If set to NO the constructors will appear in the # respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. # Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief # member documentation. # Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting # detailed member documentation. # The default value is: NO. SORT_MEMBERS_CTORS_1ST = NO # If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy # of group names into alphabetical order. If set to NO the group names will # appear in their defined order. # The default value is: NO. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by # fully-qualified names, including namespaces. If set to NO, the class list will # be sorted only by class name, not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the alphabetical # list. # The default value is: NO. SORT_BY_SCOPE_NAME = NO # If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper # type resolution of all parameters of a function it will reject a match between # the prototype and the implementation of a member function even if there is # only one candidate or it is obvious which candidate to choose by doing a # simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still # accept a match between prototype and implementation in such cases. # The default value is: NO. STRICT_PROTO_MATCHING = NO # The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the # todo list. This list is created by putting \todo commands in the # documentation. # The default value is: YES. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the # test list. This list is created by putting \test commands in the # documentation. # The default value is: YES. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug # list. This list is created by putting \bug commands in the documentation. # The default value is: YES. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO) # the deprecated list. This list is created by putting \deprecated commands in # the documentation. # The default value is: YES. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional documentation # sections, marked by \if ... \endif and \cond # ... \endcond blocks. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the # initial value of a variable or macro / define can have for it to appear in the # documentation. If the initializer consists of more lines than specified here # it will be hidden. Use a value of 0 to hide initializers completely. The # appearance of the value of individual variables and macros / defines can be # controlled using \showinitializer or \hideinitializer command in the # documentation regardless of this setting. # Minimum value: 0, maximum value: 10000, default value: 30. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated at # the bottom of the documentation of classes and structs. If set to YES the list # will mention the files that were used to generate the documentation. # The default value is: YES. SHOW_USED_FILES = YES # Set the SHOW_FILES tag to NO to disable the generation of the Files page. This # will remove the Files entry from the Quick Index and from the Folder Tree View # (if specified). # The default value is: YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces # page. This will remove the Namespaces entry from the Quick Index and from the # Folder Tree View (if specified). # The default value is: YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml # will be used as the name of the layout file. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE # tag is left empty. LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool # to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. CITE_BIB_FILES = #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated to # standard output by doxygen. If QUIET is set to YES this implies that the # messages are off. # The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES # this implies that the warnings are on. # # Tip: Turn warnings on while writing the documentation. # The default value is: YES. WARNINGS = YES # If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: YES. WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some parameters # in a documented function, or documenting parameters that don't exist or using # markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return # value. If set to NO doxygen will only warn about wrong or incomplete parameter # documentation, but not about the absence of documentation. # The default value is: NO. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that doxygen # can produce. The string should contain the $file, $line, and $text tags, which # will be replaced by the file and line number from which the warning originated # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard # error (stderr). WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag is used to specify the files and/or directories that contain # documented source files. You may enter file names like myfile.cpp or # directories like /usr/src/myproject. Separate the files or directories with # spaces. # Note: If this tag is empty the current directory is searched. #INPUT = ./headers # See Makefile, src file generated from Makefile @INCLUDE = doxyfile.in # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv # documentation (see: http://www.gnu.org/software/libiconv) for the list of # possible encodings. # The default value is: UTF-8. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank the # following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii, # *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, # *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, # *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, # *.qsf, *.as and *.js. # See Makefile and the generated doxyfile.inc #FILE_PATTERNS = # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. # The default value is: NO. RECURSIVE = NO # The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # # Note that relative paths are relative to the directory from which doxygen is # run. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded # from the input. # The default value is: NO. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories use the pattern */test/* EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or directories # that contain example code fragments that are included (see the \include # command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank all # files are included. EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands # irrespective of the value of the RECURSIVE tag. # The default value is: NO. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or directories # that contain images that are to be included in the documentation (see the # \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command: # # # # where is the value of the INPUT_FILTER tag, and is the # name of an input file. Doxygen will then use the output that the filter # program writes to standard output. If FILTER_PATTERNS is specified, this tag # will be ignored. # # Note that the filter must not add or remove lines; it is applied before the # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: pattern=filter # (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how # filters are used. If the FILTER_PATTERNS tag is empty or if none of the # patterns match the file name, INPUT_FILTER is applied. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER ) will also be used to filter the input files that are used for # producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). # The default value is: NO. FILTER_SOURCE_FILES = NO # The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file # pattern. A pattern will override the setting for FILTER_PATTERN (if any) and # it is also possible to disable source filtering for a specific pattern using # *.ext= (so without naming a filter). # This tag requires that the tag FILTER_SOURCE_FILES is set to YES. FILTER_SOURCE_PATTERNS = # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that # is part of the input, its contents will be placed on the main page # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. USE_MDFILE_AS_MAINPAGE = #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will be # generated. Documented entities will be cross-referenced with these sources. # # Note: To get rid of all source code in the generated output, make sure that # also VERBATIM_HEADERS is set to NO. # The default value is: NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, # classes and enums directly into the documentation. # The default value is: NO. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any # special comment blocks from generated source code fragments. Normal C, C++ and # Fortran comments will always remain visible. # The default value is: YES. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented # function all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES then for each documented function # all documented entities called/used by that function will be listed. # The default value is: NO. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set # to YES, then the hyperlinks from functions in REFERENCES_RELATION and # REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will # link to the documentation. # The default value is: YES. REFERENCES_LINK_SOURCE = YES # If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the # source code will show a tooltip with additional information such as prototype, # brief description and links to the definition and documentation. Since this # will make the HTML file larger and loading of large files a bit slower, you # can opt to disable this feature. # The default value is: YES. # This tag requires that the tag SOURCE_BROWSER is set to YES. SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system # (see http://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global # - Enable SOURCE_BROWSER and USE_HTAGS in the config file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # # Doxygen will invoke htags (and that will in turn invoke gtags), so these # tools must be available from the command line (i.e. in the search path). # # The result: instead of the source browser generated by doxygen, the links to # source code will now point to the output of htags. # The default value is: NO. # This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a # verbatim copy of the header file for each class for which an include is # specified. Set to NO to disable this. # See also: Section \class. # The default value is: YES. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all # compounds will be generated. Enable this if the project contains a lot of # classes, structs, unions or interfaces. # The default value is: YES. ALPHABETICAL_INDEX = YES # The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in # which the alphabetical index list will be split. # Minimum value: 1, maximum value: 20, default value: 5. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored # while generating the index headers. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES doxygen will generate HTML output # The default value is: YES. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_OUTPUT = doxygen/html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). # The default value is: .html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for # each generated HTML page. If the tag is left blank doxygen will generate a # standard header. # # To get valid HTML the header file that includes any scripts and style sheets # that doxygen needs, which is dependent on the configuration options used (e.g. # the setting GENERATE_TREEVIEW). It is highly recommended to start with a # default header using # doxygen -w html new_header.html new_footer.html new_stylesheet.css # YourConfigFile # and then modify the file new_header.html. See also section "Doxygen usage" # for information on how to generate the default header that doxygen normally # uses. # Note: The header is subject to change so you typically have to regenerate the # default header when upgrading to a newer version of doxygen. For a description # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each # generated HTML page. If the tag is left blank doxygen will generate a standard # footer. See HTML_HEADER for more information on how to generate a default # footer and what special commands can be used inside the footer. See also # section "Doxygen usage" for information on how to generate the default footer # that doxygen normally uses. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style # sheet that is used by each HTML page. It can be used to fine-tune the look of # the HTML output. If left blank doxygen will generate a default style sheet. # See also section "Doxygen usage" for information on how to generate the style # sheet that doxygen normally uses. # Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as # it is more robust and this tag (HTML_STYLESHEET) will in the future become # obsolete. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_STYLESHEET = # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined # cascading style sheets that are included after the standard style sheets # created by doxygen. Using this option one can overrule certain style aspects. # This is preferred over using HTML_STYLESHEET since it does not replace the # standard style sheet and is therefor more robust against future updates. # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra stylesheet files is of importance (e.g. the last # stylesheet in the list overrules the setting of the previous ones in the # list). For an example see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note # that these files will be copied to the base HTML output directory. Use the # $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these # files. In the HTML_STYLESHEET file, use the file name only. Also note that the # files will be copied as-is; there are no commands or markers available. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the stylesheet and background images according to # this color. Hue is specified as an angle on a colorwheel, see # http://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors # in the HTML output. For a value of 0 the output will use grayscales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the # luminance component of the colors in the HTML output. Values below 100 # gradually make the output lighter, whereas values above 100 make the output # darker. The value divided by 100 is the actual gamma applied, so 80 represents # a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not # change the gamma. # Minimum value: 40, maximum value: 240, default value: 80. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML # page will contain the date and time when the page was generated. Setting this # to NO can help when comparing the output of multiple runs. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_TIMESTAMP = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_SECTIONS = NO # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to # such a level that at most the specified number of entries are visible (unless # a fully collapsed tree already exceeds this amount). So setting the number of # entries 1 will produce a full collapsed tree by default. 0 is a special value # representing an infinite number of entries and will result in a full expanded # tree by default. # Minimum value: 0, maximum value: 9999, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development # environment (see: http://developer.apple.com/tools/xcode/), introduced with # OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a # Makefile in the HTML output directory. Running make will produce the docset in # that directory and running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_DOCSET = NO # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider # (such as a company or product suite) can be grouped. # The default value is: Doxygen generated docs. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_FEEDNAME = "Doxygen generated docs" # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_BUNDLE_ID = org.doxygen.Project # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop # (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on # Windows. # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML # files are now used as the Windows 98 help format, and will replace the old # Windows help format (.hlp) on all Windows platforms in the future. Compressed # HTML files also contain an index, a table of contents, and you can search for # words in the documentation. The HTML workshop also contains a viewer for # compressed HTML files. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_HTMLHELP = NO # The CHM_FILE tag can be used to specify the file name of the resulting .chm # file. You can add a path in front of the file if the result should not be # written to the html output directory. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_FILE = # The HHC_LOCATION tag can be used to specify the location (absolute path # including file name) of the HTML help compiler ( hhc.exe). If non-empty # doxygen will try to run the HTML help compiler on the generated index.hhp. # The file has to be specified with full path. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated ( # YES) or that it should be included in the master .chm file ( NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. GENERATE_CHI = NO # The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc) # and project file content. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_INDEX_ENCODING = # The BINARY_TOC flag controls whether a binary table of contents is generated ( # YES) or a normal table of contents ( NO) in the .chm file. Furthermore it # enables the Previous and Next buttons. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members to # the table of contents of the HTML help documentation and to the tree view. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. TOC_EXPAND = NO # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help # (.qch) of the generated HTML documentation. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to # the HTML output folder. # This tag requires that the tag GENERATE_QHP is set to YES. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace # (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual # Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- # folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_VIRTUAL_FOLDER = doxygen # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom # Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom # Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: # http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = # The QHG_LOCATION tag can be used to specify the location of Qt's # qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the # generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be # generated, together with the HTML files, they form an Eclipse help plugin. To # install this plugin and make it available under the help contents menu in # Eclipse, the contents of the directory containing the HTML and XML files needs # to be copied into the plugins directory of eclipse. The name of the directory # within the plugins directory should be the same as the ECLIPSE_DOC_ID value. # After copying Eclipse needs to be restarted before the help appears. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_ECLIPSEHELP = NO # A unique identifier for the Eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have this # name. Each documentation set should have its own identifier. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. ECLIPSE_DOC_ID = org.doxygen.Project # If you want full control over the layout of the generated HTML pages it might # be necessary to disable the index and replace it with your own. The # DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top # of each HTML page. A value of NO enables the index and the value YES disables # it. Since the tabs in the index contain the same information as the navigation # tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = NO # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag # value is set to YES, a side panel will be generated containing a tree-like # index structure (just like the one that is generated for HTML Help). For this # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can # further fine-tune the look of the index. As an example, the default style # sheet generated by doxygen has an example that shows how to put an image at # the root of the tree instead of the PROJECT_NAME. Since the tree basically has # the same information as the tab index, you could consider setting # DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # # Note that a value of 0 will completely suppress the enum values from appearing # in the overview section. # Minimum value: 0, maximum value: 20, default value: 4. # This tag requires that the tag GENERATE_HTML is set to YES. ENUM_VALUES_PER_LINE = 4 # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used # to set the initial width (in pixels) of the frame in which the tree is shown. # Minimum value: 0, maximum value: 1500, default value: 250. # This tag requires that the tag GENERATE_HTML is set to YES. TREEVIEW_WIDTH = 250 # When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to # external symbols imported via tag files in a separate window. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. EXT_LINKS_IN_WINDOW = NO # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML # output directory to force them to be regenerated. # Minimum value: 8, maximum value: 50, default value: 10. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 # Use the FORMULA_TRANPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are not # supported properly for IE 6.0, but are supported on all modern browsers. # # Note that when changing this option you need to delete any form_*.png files in # the HTML output directory before the changes have effect. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see # http://www.mathjax.org) which uses client side Javascript for the rendering # instead of using prerendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path # to it using the MATHJAX_RELPATH option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. USE_MATHJAX = NO # When MathJax is enabled you can set the default output format to be used for # the MathJax output. See the MathJax site (see: # http://docs.mathjax.org/en/latest/output.html) for more details. # Possible values are: HTML-CSS (which is slower, but has the best # compatibility), NativeMML (i.e. MathML) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_FORMAT = HTML-CSS # When MathJax is enabled you need to specify the location relative to the HTML # output directory using the MATHJAX_RELPATH option. The destination directory # should contain the MathJax.js script. For instance, if the mathjax directory # is located at the same level as the HTML output directory, then # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of # MathJax from http://www.mathjax.org before deployment. # The default value is: http://cdn.mathjax.org/mathjax/latest. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site # (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_CODEFILE = # When the SEARCHENGINE tag is enabled doxygen will generate a search box for # the HTML output. The underlying search engine uses javascript and DHTML and # should work on any modern browser. Note that when using HTML help # (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) # there is already a search function so this one should typically be disabled. # For large projects the javascript based search engine can be slow, then # enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to # search using the keyboard; to jump to the search box use + S # (what the is depends on the OS and browser, but it is typically # , />stream x+| endstream endobj 5 0 obj <>stream xS*r ҏ4Q04PI2T0BCc Ʌ Y뙙A5M5C\Cg endstream endobj 6 0 obj <>stream x+| endstream endobj 7 0 obj <>stream xS*r ҏȴT04PI2T0BCc Ʌ Y뙙A5M-4C\C endstream endobj 8 0 obj <>stream x+| endstream endobj 9 0 obj <>stream xS*r ҏ4T04PI2T0BCc Ʌ Y뙙A5M 4C\C endstream endobj 10 0 obj <>stream x+| endstream endobj 11 0 obj <>stream xS*r ҏ4R04PI2T0BCc Ʌ Y뙙A5M 5C\C endstream endobj 12 0 obj <>stream x+| endstream endobj 13 0 obj <>stream xS*r ҏ4U04PI2T0BCc Ʌ Y뙙A5MM4C\C endstream endobj 14 0 obj <>stream x+| endstream endobj 15 0 obj <>stream xS*r ҏ4S04PI2T0BCc Ʌ Y뙙A5MM5C\C endstream endobj 16 0 obj <>stream x+| endstream endobj 17 0 obj <>stream xS*r ҏ4W04PI2T0BCc Ʌ Y뙙A5M4C\C6 endstream endobj 18 0 obj <>stream x+| endstream endobj 19 0 obj <>stream xS*r ҏ4V04PI2T0BCc Ʌ Y뙙A5M4C\C" endstream endobj 20 0 obj <>stream x+| endstream endobj 21 0 obj <>stream xS*r ҏ44P4.C 4T06\TPR3$5+  endstream endobj 22 0 obj <>stream x+| endstream endobj 23 0 obj <>stream xS*r ҏ44T04PI2T0BCc Ʌ Y뙙A5 4C\C endstream endobj 24 0 obj <>stream x+| endstream endobj 25 0 obj <>stream xS*r ҏȴP04PI2T0BCc Ʌ Y뙙A5M5C\C{ endstream endobj 26 0 obj <>stream x+| endstream endobj 27 0 obj <>stream xS*r ҏ4P04PI2T0BCc Ʌ Y뙙A5M,5C\C endstream endobj 28 0 obj <>>> endobj 37 0 obj <>>>/Parent 39 0 R/MediaBox[0 0 612 792]>> endobj 38 0 obj <>stream xڕZvHrWhӄ{6T5.]g$յA 8xH xdF###nDwn7~fIo_n vMi<ۍIR繪ǟ$}ud!Т_my$v_w ilTx]KO܆]{jT)n"7K0uZQneͿ Icܹ3օ b8u4Z;#^p1S~$Fem9ە*L,K!c;h%g;0m:27M,;n?|HO`s-3JD9C)m$ 7g\/$ MYcQ1t1/+k C)Wgi;ܣIƘQ.c5n\َ$I/bo,Y5,<ݻ 9YH!J~[G94 /`)<Dmzzm9:NRT ԕ} \hM{Gw`'6Gh=g^Z.]qjeCYUmޑl=P@۹aI~eQX^j!"BӪ/%Y5 k~Ź-bR9#c8t^Uq U/MOy/ڿNWjOfM%ÂcO)YuP%]u]$Lg+O,k >m3ty΂,sV%q﷪f/i{)eӿ=A֫i]dŮ][v6ńh;yC=a,=i#Vu&ծ̥*v Y*eYc$o u wTʎy-ZlO\RݯSCN2"F]y˭/ hXćFz!%E gJ-TZ H؎WQidpf wVrr2 [eLzt6X!Pp#bSLG@]nM* %##g#mny3TffA;3R-}<)8*Zt|X)wcoyp8Z'ԕ}GT %/-ʗ[VL20M 俞8MU. Á:g-DDtEN][}?gDq;r8JA|h iZ+뫫ax-+X7 De%hhi"*lu~HQC] n}G%ΩoyBig"3ο=~dO8:lЭ{4x& ZS#^PuQ(c4o%5ne $d4|/CVżdI$ AKKcfE:OEhmkk)-Rl9em4ظL/cb+tzၱKEauPF`l)L{ܚ:>T[@)te-X*xl6$~aDz+4 SfFMĕZ 5Ӝ@}7L.hvl]r?O]=%>§#ᯏ鯟WL0('6W@(n$f_5U PfGn_6 W^!ΆA*6(P?N T}{Pl3޿ 7uIW$·Il &O%n,\&}(5Aʋzܭ;!+K3{ޞB cpTKDݷ5?T'mrط0KZ]/نDŐNMx"z]ѷK]g%$a&RIQゔ %Kr͟&Yq`9~" ɰ B 3||d$g^+;50ֽ]^<5HcY? ̟;W-D;벢]V`E * sj`-JUn*4Zăjsnƺ^$=EGb=x>}IC˝"^crL`z)*ay=k7/*qU' *;7Ts; =.GasB#q̰kf;eJmI[S6D8Ke3%de K[j0twW\1vin{ۦfw%Q$dP ZqHhZ[T,<.l뱃ܔW `Uߋ_("Oz%nWN'B,+-ʒP{_&hC`n{}L db=[zr*xVx~60Q͏<˒;?6SPaD&.?٘GN)Y ,@HKvEA % G\JbĤ/~ $:+U¤Nm%+S@?{)O"̙< h[<5VG-BYm|iX &hPp<INz}s%(^_ZħKLfw PMF,.mJq( 'Z5wpCp9k!L`AJvPV ,aTZ5čK!v9[C}$]!vfk t)ax8.) {!kfsM=B5(zٛJ6Hɺ"iuA$յfN\T1WqAt +Ψ{-p/((yۣbi'Vbu]F-i'Q0+꿱C9Ot@NrЈMQ jk]NOMh8@e^#{D&ђhxR6R1 mZ:d/^w VvzmAad!L BO$FRXV 59bǼóB¦Y%M{+wJ< qaF`BIosS;9=\hk'NK<@,k8mvC!/U.ۈBy(醬\$/,.qr骢eYӵ*vGsQh#XОho6`GE[ҒVJ{q-&m\ևb.W垔/O!rMm |o Oa^vQF(Hն˻T:G)aC=;j%^,2_x\Gz>!hţ'lG ]Gm\)S !K? (`)gTG^8 *iCqM9jIwȓL>=Rө4#aʷ[*7u5\}Q#7RBڙ=ѕDtJ.AUe ׽;tP޾OYSDH:*r:_MH_UHu_C_G endstream endobj 32 0 obj <> endobj 31 0 obj <> endobj 36 0 obj <> endobj 34 0 obj <> endobj 33 0 obj <> endobj 29 0 obj <> endobj 30 0 obj <> endobj 35 0 obj <> endobj 39 0 obj <> endobj 63 0 obj <>>> endobj 57 0 obj <>>>/Parent 39 0 R/MediaBox[0 0 612 792]>> endobj 65 0 obj <>stream xڍ[IFWmP}jMj(#\'@0##@.Gн9ܸ7jW&fq?ݤ.snofߞn7~9]5yWptn!oW"EguUC6v$mg>fË XӻN ̮p[cst꼻RPJ˧_*Ɲti}W?ɷy4-JQZ}YPR :][}_5f҈bw5`e 8ui+}[~=]OPuՔy'p^]Y<Z' ]A^-1=NG{h܏^P^L}Wt;6{tNƒy Eh N-؀:StW%S`ƛH=/Fw/c CקBףe#Ȯzjd?u-(^v<΂δ_[ݭ:〭a@N{8`&I# tўvUc~CuҬ~4C|糲6u.nlK<wMļNoϡ+9?':\Y/˼MU>V]sp]~h;fOIDH{^|\o5;p<0uU (wυn5ֶ!z%︉C/AxCtVL4J]u2B<7[mmEBw.8]/t:؛ޱ4қfyӅ|WP_vF.4}/S#O _N@ʍ fQΤ+3W"2{DZ3gA@e)?|H.oOfu=Ad2\|C*| sջ72ॏhm.͹=,%L߭͝!fQN'0Vc1{,=0XWUAP VoGq| ӴTpR t;> +xx+늨Q MĞPӫ "RDZм=?Cd'7%\$cvIZ?AZ-`^+ӉՓ>( 0V:gE9"qcI$  n1PЗe.pf@/IVuq("_YJ-Y0׀2K*;l>yz&WyԝوHɲF|.P;3K}#gJD6!l< #XOҲjzgn-hP.}[5|UVo1~Ol:gbuEΐ&R~]uV]R[: 2Ռ}8SU<6ՠOQ;Yw13},! ߵ*+Wj5[\__u} f? =!ta Tdpѝya8Zwo|Qƕִz Vz]$ h|Wb4afZ%PŞs~hA?Hy <@ b_EC 6=egD~{X,a(mfS0UIڒ3v{`.DVՂpFl쫂h_[c$YYlQ+Ɲ#H*~1upr[Ev%9m%R|)t@,T0oj,V)O.Qe Ы4}>d63zloI+SoE"N^'I>bπGyquG!f`s>r9rv utԃ'ܩewT(aƣCVG"l2CnG8|+0R,FA4F93.Q`"*7E c李΃/4Vb`MǛO_B_^&C_~8t%1x's( 8 / DژA`  iLXxRZ{8~ S%2umr5Y$ Gl.\E!*G\icLx˹^K ; tJʙUaz>e ť́HWY84!.sP(feB?aHOHGX=>*Î i'1(gޞƉH615"иXp@.&M9pQ3)`9+C6iWvx5OxD/mW"xշRPK7UIŘD,fP l`% L$OGy+fb#WиtK> 5ͣ6F\4}I$Gwdzu㗯3/W7 wA?l9NW Hl U$Z0BatF|TH '4_GHz^Bc+C\-/282H1n^W>̧[8~4]#T+\bRP[NC(J*ukR9g $ZJ"fAɕO v] c`24 ]+M B^n9?m԰$@ ZYTy9ؘܶu/ QFl[\[ZWPTfNrz9|)2!2mE>K|9ۦds򗽭O%˨* 9,T~4R(鵾Hӈ6s~tsK\ٵ(\>]?HǐSl/BלI38rbݩx^^hDߵچtfd,44vK4Fj ̌P ܂0PCiN!\$jE3R,]H80l!e$6<5F؅3&G{%ǵwoKV#\6B -ia|Ro`lH_l!B-O>mϒp~u<,& Kr)IR<}X=/ںgH8 翿z|m115h6Kͯ* OBHBz(v@G%oǦaQh}H2 5KR|?Jj1W>Z!O՜2Jf(YDM\׮3߽h96-+1Hj@s X}&Wf<,+ )9#1QD{){(K#x3x4ItuKQ5c^ ܵ:Ppϥu% K+?Y^fMo8Q caREٜ/)CslPmƳfOFgj?4!a^Qw!}|DVxũ{0폼OL%f&[ V)q$&Az6ZwjH8t-mX% *,''8̹@"=Q riފ9)Vsp*qi8UEml~2"QŸ2yhy!٨&Lj`rDпK+ɬi+t8gr /r 5J4TLfbpȍHUdj H$$p޲ħh‘Z BcJ#1vaf_K"E4R,J,oB*? D ,eqNdR%@RZNrp: i|ϛh#%#^rnvYl*ӓU`z.Cɖ<^XM9MXing V~^uq^Gie^FY$Fެ$%a/jqCk9 zY: aiHu00dDI Mw J2#E_NhX矫b˶~ds{qAגf][HF4Iiv!{J-ꋱz?rF%e 9>/*^dD&$4LT4<]fYOh!i7Y>ݪY;~TsdjމqɅ}&o#weJo8%O,*9I3܂;ْ$34xTκ.2CKZY՗0fL`LPth.trmE orr7oԻؐ-(LQ5d+ 9A8-()ĈhȂiR$&FϤ6.$6ڦT߉]Vp!S#W-LiOw90aJ?R?("NHN=Kg"o4-r1pkk"T'iӎyVw*mfM?ƼKMlN^Do*=Mc=TJCȨ\#4j:=@-0"ѱ41FbV䁕uMPب7i4o",w'I(crDq*eceWX-0{z5`fX2p<29WHsB Y,/_K4iפlnfL龕" ,~SE:HV\TEMSb#1eh Z]&q99Tk;6;DhvPyOArxK/fB=g'M 75bX`#ܜߨK:Y«Lӧ2Es_9[{GrBZ0P&[EӷuNؘ 6#jbW?L".jg1<yy`&S21nZrE!64|%̲_9RfMA 2pҗ!> jA01YfSw/e?bzI]}Ri^|b en^>*Laf8<]RxŠԿV =&!OEP P Ki<] (ԣ ˗mYc:g2e筙XKb#4_s(J CQ0jK:J ~xF"kkEW|fʟq٬l24w?eZ endstream endobj 64 0 obj <> endobj 68 0 obj <>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 80 0 R/BBox[0 0 516 287]/Length 2485>>stream xZY ~_ICù+~C٪,=I4c6dߚk=-QQ_u;#ir5 5wᇻ_%\x2l`Q^ o*$rxCGfpcg +: ?urAr1ruVK%yAG>/dzL_pVt ΠNlGQa4ąUzeL.k.j\sɐlwI.Hes\vIү &;4P7i|L]5&1ָZ=%?SjNz inƳDHƆ0k N;r'+wD' \PGvD9ѳdE<>7?n@L)L%Ru`Pꀒ+9l S)l/ qj D~eRx7OŅbkiUvc#T5Z"p47B:Z6k_%r UH84y2u9d\5t13&&ˣ'&!ؙs4\Da<:5!$3{d@n^8.75GOb,VɄDxRdDNGp[?E9YQa;̪>9͓ӴhaA@5q3OzfKG 񳰎G< rH}JKHƏ³89KRQpU3ԛ'B>K#mKp@<c)'qLbKHMƃXqŗ2;9J8\DzbI.(i躪cBHobomNZVop%9W3m#9YGxGR+=ia{]ʔli?o%`-wiv MF1Rz:]-!d|=9xr<o$H0Qm:E4p":dޘHM mœj~a"!x Վ{B8VX7lBDt0i{bmǩ6,-~{$Gr fk-'{ uL,^+Q Z?zܘ3=t}0! +#=Qen>l+g`%z6R&o֔ F@D7o[{3wJHQ)Z4Yb.' 8;J >Ǫa|<tJr{P3@zMNޅWn]5{Q' 0;bÑ*~vNHFf9pI .zP=sJ@dˤ`W̚JxIF\?Ctأժ1*DZgC9MnK]r|TvAq |>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 86 0 R/BBox[0 0 577 194]/Length 2745>>stream x[IWY5I]ŇTŚ[TFr!?_ rju7wI 5I+o˧狚.R 39d(?唒BG3)j2 Z>%NsK7?+]Hf&<'%g=da9 {۲l0rBE!OŦ^LT|+;%>G,ewTtOsR鈥"LFYɭmIi~SLaSQNTjk(㴪VQck&6³@FvXjPAʌBltVr+w;Zk5Xj r#m,lzXYf\Ǡ0)C HKfeeeF=ŌiHyX<8Pz〣 :8~bM O ؈^ϔdc#,Qz<e|Dꆒ̎v\f-LYY\mtD]f$۶”%> (LXMYSEjTq1#etherHհkS,J|5V!K+Y#tVV"՘|d[Oi - _@%}0.FD9\Yqd 7tNVVzVgYzB~iVat톴:i&g6B|{yS^yINuzR`k7. `7Sk8~%I`DCng!tz g\XG/\%AKdϯyM6XMŋo^zK3~k9q-c*v'nmF]Uf l$ 4׍O$οGLj$iB4tBAF Bnu\ssmf9ȭY ݗ/yuN/h|_4Y%K)t ƦnN ܒu d l4^,#K*P?ytƙ|Z%% $0sA9wfFs;(lAyz p% t0)Ѡݍy0䎼}&̨oSƋ;/_?;DtMfAI<7({L-/>!w>}HJՇώHKe2>i9м'e0s_d7g3׭tO=&r6o4> 9IQ02mHRmFIԾ(f*~"}ݺ2^ Mi 6<6W*eqzټ{RFMJW٤GiK ev7;^˳Ҵ(z¤7xS4cO#".bYH L>(DŽP)`AI;B|"o D𔂭Xs?u ˆD~pQb>Լ8j%lU1O+؈5̶i#t`Ly oP)(6`JozjCTkPU,,!z7 \{ZuoU A"MΡiG611۬nG8 h0Dyc\2[_ mh |BF/?Mp)]NLiZef)SޜN=hbh*3Emje bwCœ1GriOcyAg6l z0׉ᤆ\rkU mNY LD60Lmj95*o~N*؊RS endstream endobj 87 0 obj <>/XObject<>>> endobj 58 0 obj <>/XObject<>>>/Parent 39 0 R/MediaBox[0 0 612 792]>> endobj 90 0 obj <>stream xڽ[Y6~_QY@,:whX;Ӱ쮈,K>?idtz8M~tg|L7e OE|GSfyBޤi,hν=&:ꒌn0GeapiVd\'.lƱz"8.o4CU|Zd@XW g~kU7}V_,}uJCԍKĂXo~5 (8L`74O3W4pvvWe&~<qjc][ !4󥩑iLZ-fv7HGDq8朇zhNOÈtOfxV'"/ ܅+l_P5suvn`2kPliУ*~C? ms`s3؞ ӥ xg{d~KjNV7}da=2L/ȷVkƷˀ/@Il5F xCZF9=%pǿ5 nG<*}4Η[+_lj&)d. y@2I$r:燏)ΓhTtPhY0x&FHP}3-\T&ˋf `R(C,l j T9-v,?qe3T^>;!c#m 2OsouCTuÄ[6`R>bX^zؐ+P$OTN r&.tK(N_I˰w=AhTY qʀjg~jokÓEh"i>6=UL .@6;=co-փ~A <܈jM=C.kE(x)/F6P` {nRlYՠyEP5U[=6V`)P ۶S-bփ0Hu[b?NLSMz_(Q<ӨT2Vq 8P?ß=ɏlHix07nr[ڊ&Rڛk`5 .칅x@ hON2] J#e ,pt *ICҍԈUT@=Vfet>cj0n맆ao((`)@@F*-(vS$M3{n D,\wq+'6jd9*0sa" q誦̨(& g@H$ 99ĶnM"VwlDjDKANb>AA_hQ ŋ(aҎ{` ,&3BjX-tP)ւ)^B(ak @ M|Cx{p01xhrfKd5S3E٠fˬ[HT+N#M̟}S}[a-A x=K4Q -.=j ]IWg3!DF^(68D֢͌Cİ^֬eXWqz%n_2BXJcaQ6 j&YJbv?P@pvI 49H'=l ;| ;+ix E=ƄgW6 3mW> W1[R wrDt-(LmM^y9ki$*Mrzx#&2 .AF`i褄럸ر ,,xYP%ÿ S q4ܜ'/A P;H܄`;BEDVHx[=D Ʀ H~:u',Dq/T,~_m%uo|';\vM[iޓ |YRX-/q3pIч\DԚ֒2J^>r#m? ?p~8MEVU&+&-h*# 82(3#Öcuphا ,Wo=Rfj\9bhBk3  ($.0Q`l8ge͙ 1>;sJD籌d/.O6 "y2Ϣ& ?%b3˕d?_v 񶚺R l yg 92/z[+++Z3yT\XB3 ,z v9aOAvs| n>d]ݽ]*RN*ԩû<{r.#ixG@L&" ._o}f/$BBH_+ȄqYRcywB {8X"k8cʠpK#IPn 1r3s[wJ$sќ仛4&iKoh[i\Ξ¤\\;C( `؃éo\֥2~^RUBTzeGJgoM[rP-@~ 䄎[*J,JPj$P)ƒ+Hn3:=)B+;J8核*1WPWb%dl$oCH&=^+ { M]|/B"vUi\ 5޻%; gKg9Cou8'C/ z8ondW[Τ@6Yuk*2 Kܷ8veSl tkY4O; Xn gz?*KĀDh9ϖcgW1UplZ01Pvx{d[;-ֹ r ˗E"{X|S'p?zue؆z_$ $xz!y#ق;g(eܟɢD{jW,=v^;[`[Xn.:4J,aP@i֛$%7C?_|̢0G ׾ ^Apzk`wfAa Q#a$wk$S >*DR7*SdXr4@ȗhc҅B9T`^Λw-^(bhD~3G-zԜeǟUJ7Er|P%Ǯ! gMXhmҘbE":R>WߣBdSn-W4ryZLY~_Sokw\:*Ic Յ ,EcOD\ޛ$mqC:6)>n( | J7L/KE<:aEoaujgN ukylou4f?>1 endstream endobj 88 0 obj <> endobj 89 0 obj <> endobj 80 0 obj <> endobj 69 0 obj [/ICCBased 95 0 R] endobj 71 0 obj [/ICCBased 96 0 R] endobj 70 0 obj [/CalRGB<>] endobj 72 0 obj <> endobj 97 0 obj <> endobj 99 0 obj <> endobj 74 0 obj <> endobj 101 0 obj <> endobj 79 0 obj <> endobj 77 0 obj <> endobj 104 0 obj <> endobj 78 0 obj <> endobj 75 0 obj <> endobj 73 0 obj <> endobj 109 0 obj <> endobj 76 0 obj <> endobj 95 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurvzzzz'nn///bbG`V__Kww?33(3 K  c  z   " g 6  [NPfD~8":!i5PM%= UI=1)& A!X"p #!$ "%!#&"$'$%)Q%6''*&+((+'i)v-((*.w)+/*--13+..2K-S/3.G05/2261$38N255946;5J8=69>7:?9@<BBH)@D$J7B4EKCG)ME.HODFJJPHlL1RIMTKOVMJQQXOS>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 100 0 obj /YTUXND+CMR10 endobj 102 0 obj <> endobj 103 0 obj <>stream x]An0E9Encb7tâUƱ,H}@]|Ǐy0rnޗ)}./6ݗKƪu?AL8W5_?s@.kn>oL|cKsu4&K UvR60ƈ 1S.0hmM>0hw!0hxY^*A!s`rD[}favH`ɠU\=(F6#"ld3uE6B^6B#3 lFV}6BFhd5*l*9GAr^ts hdU[.%{T˒ǕM 4~qJ endstream endobj 105 0 obj <> endobj 106 0 obj <>stream x]An@ EbnH7&VU S B}?I]|m'Oy:~ ӵN_?4X^ZZgmjm1F=欅 ݿR]'.ZW qETǍ2@q*fN`rlAq<9(l(j嘔A)cM1+P 7 R ';,䄂J 'lvU+ 9BtAW\_*m"܀ȗmHya}`f' H ~r endstream endobj 107 0 obj <> endobj 108 0 obj <>stream x]M! FLӍn\hzda!Cp\x{iG]x$Z~fݥ8XQBY<+bR>cr˪]2Z@aSsj3GvK7Rk\Rd`긆O5(-V(; UE`1cP:G ز.Qb]W 2w@^J&p1/>stream x{y|eTU_髺U]}$}Ns@!%A^TFVD;"Y}_gtYqUG:~tyQwg9ꩧ~T@ tu.b|T,Zi`x x:xuM*xz#6ve)MkPcj+Mۮ)\G8qB#ߵi5+Pr妵)TX7oݖ7 Ӓ} %(0U6)%;ZY_"j ?~G}~ ΂Ut^π lFR` h`Ȁ1l@ j}:,p|4epQe`5vǠA{] |0\ l[}/@rPˁ =|?;'߄*rp -K5PVU{SɚD2,&'[#;S/{Ŋ7d\`{ ǃsߘVJvhAoQ#ydpb23T:=ã8jW˚J[7ͽSTIYzCFtYx <CC3l(̰̀&@D:ЍzBElw4,RɳQFN\o)yZ",͊,x0##t8 Ie u!c'bX}-ˑvG(@W}D%) (,}a9=%Ƚ#CBpH#c/B7:`WWӯvͯJ?r^q|J#A#6A!{{ᅲq]CVNt͝}~s?wjgӃO=M:%>Ua̙3Cg>= lSj'<$ NCi4 (yB=R=yg'9V8pVGG6?r# XŮ5|zq稳ڸBOqY'.;oDcd2,nn= ۸܍{j9EzOap%j>0L= w9uî9]^nvy5`.:Նšrm ^^h]p-Vh2L⬖g2R}crՆ8-SD*TKpQ ҴSJk62iUꤡҠAF^/urs %0=3.;; wd]Knf[d[(o^w Zq0 B1|L ;ked}&8@}?sMG7oI- ~/KfG] 53mmm[Kmm Gm+lZYv< +V(@?'!{)*=:Ώ F}9$"H2ƨg%_|Ghr7*/EK}ɳO+4IZ~!< = X7J)7S>&^x!7>|nYn0'˯cYGJAKPu0/.ٳ(73!K菣%O[ $5SjPr?} ?; u0Ir.oTQ9jp݉PVt#&a,f=ujObW(1=GV^fvW,Mф=*UO~T']{B jUJ%*Đ-a]\Lꯪ.h$+G+V@h|n:A^hcz[̦tw|`mǎ 5&t$V`mV(GD|)\|po+V=\VԾtT7V%M6<{ ҋ:Z^&u9;7"vcz\#֣&C؏kiB.uu!Vp%sdFrIfTV7Rn6"Uf{vlL4n̐Q]UW[{P:ũ? eeɪ{QRiel[߽sU!Xb Nbk< +) ߰U!60yAR*d2)*!HɃ+2HCH aDFFVVP "l ";TϝrA{1/gyq$I8/p|I!_W[RӝR抰d'ya\ ى!=/O\\/C/-XsL`P.Y>dѢRBK".2gsq{=lຘ.R.dApdCp!&_ EN%Њ`><'٤~4*-Q9# `Ul2~_VWFpx\J/f+ʇ* rHWuV]%k*D|90!\8Vn܇zш%a9a+@UEAmV|RCHŹ߾DC$xU0>:Ե/|ÉG/ǮuM̢TWzF]x,NG q2.9 BЭ.3FZ4E_~g|kv[J&8EBYe&ShmKYXVexT1]R@HA!uh~UAUU޻LЏ8p?1BPfa~eEwBr H>|*-ZE~Ύ sΫNۺ;澱V-arEG$_HT-~+yՇl^lB i]-Qໍ E"@)1+%RIߍnS!"KSTjR˨ u. ZȒ*MeJB ?VO|f޼0V@z>#H]xU9WA?3: -LKcŠ[)ڤYDUbWTA6IRՈ.#) \/:ձ9ζXsQl:Sٙqfb;NɠlzP7hʽ5WrnGg/|#t@ 5-SyZ߯*ܤwM~y S8VK]z]s8Ͽeˁc:fjU^Y#?<[Y\珮}n֏>i +H?Xw~.wf6/?y͐kNQv2sM)x)6{*Xf&$_ + ve-4;da%̄Xte 1wշ9"蠎ź#ct_3wxZ}!xJ&edd>I,:1@04d}庵7Os#4{*^Æ~MŜȝv]%l ֔{4ՇPq_M3CP)CU2<].$= )((pf5=(4zv#ʘ; W^bK,ԋ-AY#}L* aaT=TGhh֗I\Jd5> g: !Fܤ Cs_,aw"/ "}(a&,R,I(8S1e\8xmj?Tbr{j,jc8Jj7,qyytW"iS jJMTDMTDMTDƶvrxyD +lfqކ8Gd/8TL)4'IPP0aVt~Ztͷn_V*li:ѡ=;jݡml([Ba !:xX˺p+u:vMhr9M<陈6"D:#t|#ɬ9S*HtQ)hO %@pP2.脞"O5jg>~=щ}74Uj4w~cO{+2gIԧ\yoC]p^j(\QK,jE z3i'8=o׈QF@ӊeE؀m j]>6jIF^Z4rUa I`P]/DC>*b1g &xm-{1]Xz({>g'~=(zE*8`grv9o*xohdJ'DQ*wSTQIʢ'k?B::@?LP@R.Kddtby.fgS펒'zؙDIȉx&E3LjwwCoϠ&șD&Bة1?Q2xTY"{ /_U ovc=&Τ@75#o$°a_RCB>y!βtS{yD17}5B.V,y&'n\;.;̬Y;+Ğ]hv6TFoJH|L@X2*}_ڼ߄jW3%yn.)(4¹Z$R"TcDphBhI#ؒ jWݙ`bv-QuL>x*V޾oIeWg=ڹM5"|. DEI6:\@vTmu*MtĄʍ14%As$x{ @u[2<9n,r7MG6MH|V#"037hH(>,=>CMfV~V~v9F,҂m@Z ʌo+> V{G&=qq_,msZ")fD9&'奋c=7x-H[H]7CԮ7}|m na쵹> Մ CKJ;kș]e J#>SxF>|6Vx@]5دx&ۛQgNB jlNS^_Z2`|$р?p~5իS{ ]l_W;vau_[¾u}і ժ;;qݱRj-6$Xg?o{.Ǘ#wo;t~ܛ8n9V}alb_eA700!aCqF="rha M.$D$ˑ ΉGBC8 Cь'RX(!]CQO@w*ajD}/R71PL!|[Lr>߂0MHD@anŬE>gĂCj++q8{S )Y9~̈˗֥wށΌ:Pcb`n&^Dʅ`iݔL(`|$]UZIo-uj}쑜2?E<$`Thϡ'CP%"&8 jȔ(YBlS`=bELM )ɼ{W^@C{> {& { H=P$p8roiNYHU6Vr5Kx^s_Ñjv,!o^2giF2isNՙ]E/`Fg7Ȇʵ'Rڶ3WouN=_R͔TV2VWJR#WmckxSt⌽h~?QpR,LyiOɸ 0J, ZqH4Y~wϖF=M:\D:qxQdc*vHfA܍~D|u 1DC8Su}f w1;̯%R2Z;ŭ)x8;xO?m|D5'5ȂnĻٓe_@tv?v!UWE"U$EHTc#{.I1TTza`wnvG`hwtkyꥹ`0&l8b^t5ZR"7V`(y08~vRM(Ҝ&۲xI,,U4p AR!H4){Z$,+ X{Ti, ,!PXP!Xܻ8 /}LmXQ({`8CGI<nj xܞL<:uWݵsոoMrs-m޷eF[\E/0zMڇ\D u7S8'?( *%n>);J6N)1yÉ8XjvD>(Q2 @ƍ$dzAY1F0nĜX<>Yqo|4M6Ruqzj/zi[w>v4WXƺw^Ex\fƅӥOϬ1$PI-5(5IAo}_QgJLgQap,Ro5BBZ'&BJ#F|8uq=q_NHNH|LHԢ.G'Z n5Oj8j_OGL퉩 kSW-ꠖlt1xn/$?\i=Q& 64=r,7c2o%PV<,wWޒ>n==K4p$] --WVY>S^-Tt$j:Ep*ⲙmЬ:s#[ǹ+s0r5x·-mU2EQJQP2\ҫYAv {D4Uᴱ`:ao;#hۆr"u%:Ta4)jNj(wH`|:9J.QTJL$(3( `&fVAv.Ej,i!34njzݫƿ>Z,~hJ`=0׀+ @dASꙛ:F݃?a䠋~A;sov`aJAyH^Mn`f#zd*P<f0]xY`̦jӟM~G8}چ?{d}Y{qWQUAx>w endstream endobj 111 0 obj <>stream x|y|Tsνsgܙܙ$sg&+@,  Ā`*lui *2PP[Mh} sg~~_9ٞ{s +lL^4CW,X1U_zd>o^jdװ@|%sw/%8Gd&W.H>+0|Lᷰݫzֿ30wL2'R 8DHg^ipa!p:n_?0%N tP %S<߀ #M##e /IMOx5Ј<f!p^{߀'GLāPHL#i%n_]Nq.½w_h|GsPm 5k|@H82A\9d7ЧhrGy0>)"`|w41uPhԦЊk߄| vdv #^Ie B"]d+$(OB>EzGtqp{7x/|;V:Np^?9bc/ߍ0'%>VX4VE؁4>Q_!/8b$>8ɭ鞈"d^K2a9J^#oɗ B+I;prw#z[_MwD(I ?[a 帿ҷi5=K~A3 n 76* j ۄv}rހΦvx<g8Դk7v:ג`=>|POjQv@J3pt+5qro!-MFnU;!}c}Z,Orx-{ a9=H~r@{L!p#҅ܨe^U4<D\hgQ6Yt.&q@ uz"ܯy'8%gQJ/ 7{!rPk*@G P^v5.A&C'Mz9vIPi]./E % MͽÝI&y3&qG&W|N+`' [ ^RM JT+_ y/zO"w cs`2Z?C]YQ^VZ2rDq‚Ȱܜp(23>q$.Ҭh : J4=Ǎ+d|?d=P8Q]CS.Ts%jS=ID }1-ؾ1ܪDDUk[ JwI%JS%=M퍸\n4@ɌM3^%ZzFR0X/Í(4atԖF0ZX% Q" 4h Qe){ؤ=bY^8nKʞas嶡~ɻTaݞ۔-Cg x/nniGoF&N--Qr >Rao*~&6ҾLcKzh|=Qv]S'NIFLn*3=4[amtZ:kMvQ ",P0SгJBҨGQ]VzԀpOLOΫHt0")X ѥU o緎.BLb*t`'ajK@ԢHkÃ3lfè{m\QCMt;wLw&'LO:EiiOvŒzsViE6J9<2X|6~MJm(Q}\yS,qݥUnK_uQ",= 3f.kFV{{:Šُ HNϪA6͛[%ѨƩ*8}v~>gaj>Ƀo$~6NB=nE){T7 nz]ap5hڝ5rV3qa]Xaa배-v D&6rG9#u\ {/+޳[e ,ۄ[we`= 뭩nS_UVo>/&P|GN_kWkm#vH^AAAADHpm8 Ƿ6m|m`~jT>;5z];r]72pKvn-\ͮ+J]Rmm rӸTnVO g$2ȍq܋s`=kaqgX1P\ poDFdقe;<ikīzPq gT8:jZA*Wc5bU㓪Wոr5sJ,z•A1,XtNWta!:̅6tfa.n F3=Ўe tOaaeXŲ 3:n2Gjuɨd-k|dS>^]5ۍa7Er+5T5Q}sZ_k}h֖IssEs1ҡz""֭먈,b-X#z;n^g~13Ȭ>+#s h@'{7>x93`uS7'YUgŁHN > 0<zQ GR?\_^28 !`1 ` )8FXU E  i CM!%p0VcH1&㍖G&iъȄqʜ^Bl^n# 6t[VvOK/OyS0r %'LohdDFb :q?-gUk~Z464^CU( K2i9fU/S˼& h85,g8%qCp'!''{d[!ɻ k=ԉ=%E cVwÝцp;~Gײ^X4w7^غ{aO_;lYl>[l{اlz,{YOgUjҴcZ1lBn[ǸU6W7@~ fL-baSl M^j{CU25%=<MKݝj|oww+f^ tCj7h-ޘMX6k>n] L[m5?Z+Jݗ~fD Yp5⚔teZs~XCsJ@dz&Ag9z6,0y72Id%TDrshmޑkIsjHdR%_uJ^x'Ᵽ]689>Em}2}<?<,3"f-9Bc9#uϓS(j31M_11c:1g/z+] L7P@" O#@!O$FH:5=𺳕WЫ3lW' đxc.i-jjveY嵴4'BJG% VxkFuWE{r\ÁA&5K&-6=w3/&>Q߽@,Z2rd}teY*a}֊{xɊלIox>pDݷW 1`:6 6s$!!^ЅATr dnEqsjJҰ5bĪ\5\zz%]tENog+KtdCRPWGM]1ggȟra6oDAp&%MQ?&͑;e:JF~!C%sH[*w'7|b 1w3W-.㎣&릅̚PWѷެf `4fhbbm; `7!1c7ds` \NTk|7F,x| " #i?`,8I*FCmˡspƐl7XOr3>%>ʼn N/,Cy> iVlfٗOvE:S J[[W5'P4qR/#i!m; r^UWoUC,_Y9sgf,*nT-H 2v:dlp(cN}ƒj?R gID95qy0 wSwUj|7Ho=SW\ZǷ?3#\&"a{~3|i;4#>a 3/( NO '\X:lSI3P * ,<1gVPjuق(X|4+x(x"h*<[_)<*N ;,Hf$2quK\Yg&]Ԇ3\0K3 Qyh-@J 8anHƽVhBT3Jz2ř8fH[, k76-4yv7' kvqrմ˗̚N087h2\7j6^q}~grݍ쉹3ٹlٷ7 NyEtO]|1ߑ{=zq9M8CYa=7{asZu{l Ѐ@ 6cb lrȑ;/9_3 S,H&4J&vP]DHBRECPV %#0®{Msafثk/pt&_| }H{dU{5e3q\F-|wVVLcݕc'f *jWVD|H; ~n$8gb mBB_'O u!qfb Hn"Id`_)_bOڠ_SfJMgNs r %7[_| 18K ǟ5i n(7gm/$jLyΎ;tX #eU&/W?N"I!`1t!;] s/tjr)4)䌩я9ܚkoYRjւ+\=Rtx^m4L/~|깯ο~f%LI^hSK;\ݮ]hiŐt- >=`EqxHDYJTGﺘޯgW\"ՅBLնr[EZmVcSm ʲWQLw֯N׍ҏLo7 5MF7׎]S*wP So:N:N;xp4Öre4ÆP(U,KhIsQIIqQhT;O4溆.TX$d /H~XZ° |ԨlɚxjؽMr22ِCsBRW˂&=Vc=MKM{δEW( fuf=-;{m]4~f:P|n <ݰZ|+[{hkCOz>G|ټݸim`GzFV%kFIĺ=J;=m?ij/,]#X=zΨcnT2|Q}֜TFĚ_drWͦ C*[ +)4%Ӗ19c^ >CyYhvΒSfKL|1 TJX¤>"ykgI+1AG"2lSk;>_6J;>tC:3wk'v0WiK:ֻ<<+gs?@RHqi:8=j8C~ x%%l19oRV^^vV(bN? Rhʮ`qtɦ?z_!HJ-<%!5=|:,傁;/8I'm:'j!s.&dt:0Q|:4#}0ڦ$4x"t}/'JiӔ Z"'^ qޙ>'fa_x&ܞKc\](եS KR*YΣc|nc>w*l9o8N5 8]nW̦}mYK+U۫$ӝU{t!T4R #Q4 g½\USL1RS1r3**y+3L_|WPeՍSm]O='䘂+7$mȴ˶gH_8j Sɞ)[t$3+8lg*e+c㭭QqWU]ޮ],'IzA-JOTjJ.)x_@c⋾Z]:uM*1ET0]CqQ2:!!#AjϟҶ ͮytF>|KϺحNMDHEJ`.*XE KRӬ=,nUֆ,*fMɢYYee慂y` [0N6r_.! 6@ƇtF YtӴ2#H!EG%t"x^qIvSm>qu@۟< o] :y՝O@?0L6vE"ﰲ"ܮwuؿjwaw7 xBBC 9.t);t:KKƸKǖ<όSoؖe$>p ET>|J٧\Hr  9+z 7<[cDzjoK~s+8HB endstream endobj 112 0 obj <>stream xY{xՕwf4Hd˒5#d[-3`=`;$&rl'N6q @`J҄&!N$MBXZ bXvKH>.4vc۔v gFqssflBYhS#gZݻg(@>%yS?8i[%M[ܘʸq?a`IfTmûRʻu7{@nٕzFg[7_64c85EȭtLbOт툪iT so*; \H)PSxS寿Qn܋q p /^,n|µ[kmB=h-֠m(c=!4ѫ G(@#&Q3:NFlS&߇⯠{#Q29tG0C' CMv܋r˄x}J| F΢ 6R~Lً.Kr]UʊrY]%EN_hX[Ւo5s zHרhEyN</d>KHNY m|r O2H8GBS&xRjPo ,}8 'E )S'2҃|?E@f}&p+ NHJzD8r"{@! 4k}W!@蠔<,=Mx!zr=`".ǰ\DJe  Gx 5QĚ`!ECtz4uB&y9M$+:0d-Mkx6OJ$}|)9{Pu* B(4b$ح+A0'dIn$ d Z`$2CjFjy1 :TN# _|QғNc$>O$"<I=SxpQSZȾp-`/{@deb؎Kce(גjesԪh8Ee d+;8n5q 6Ex)ns6#&ym|#.k4UftA81:u+Dpb Je?W{$;= Y1!=kg!s{&07b):4 cS(TpiPr\xs pہzHr{-} . *-(139,X-硔y1aKj- 0,tZmuDᜏBC1X584x4,d޳9/e:y,QIH(H$9"$`)~ M_'zL!eF v(QnQvX} l6Bib^9ٯx EFiU[D 'y^x Z0X;SE5{ku,sAc±ǎS}qjKاOَ'O =!=A=5fGu=Jf!fYa0y+j(E&; 93+"w@<^WϚҳ9KSX|L,}A|zyhn? <{ i(?ۓ]g;B>kmlZg2r􆜬4͚ܬf z7K.ѺY-ԤgdiLRe"Ld6F4F&ZAۨmRd;I'HB@ "RfjQZvcIߊZ$I*NJi4kNDD"׬NaS5Pp LP)*H}p䗉1kǎpy5qsxtxХЄF6oeNxv.َ 4PAW$X@"B>i hrCpy*+:'Tz}:B-B~B"?Di7:22Nse$ANUtT+*HHމʙO\f>ALLYi769DM\Հ+uEKpqO*_HrL6\8p OVrqa/*ZKNW_[[ɐ l?Iv{'>^ iVtD[gw\>Õ+א8.9?SMqs8ݴd'yvZoZsI:IO!#,tU4zGIr¨ĘfX*Uɟxƴн}\a>>B8# ݍ3ܚ`WTԚ.?>NZw1e[UXTƓ6D, "*+qߔ[Un@kT1NﶅWe.XShGkÁ7^ެ^m[+];]GnxHFrR vwnwY\yli͍7m֬2s+랩i %}]jE!}YfSնl]fM閲w,##|k7/73o5L͕YyW0U^/7^%MmGL,:!-lk@uĝ&楕'3p+~35YO,ә 8MʿkܢZXId˱2؋*j»+x.lt7 vmšժ˼uzҪu#='ohhXۜVnj \KTg2><ۛqJ+5g,ҕksL+n~oK"KMiCzs][gߡrN&;rmttnx\0Gp\ĝAȾW޸r||28ST4 `/ `{#m >;F$n .]3gEH 5MbU:*I׸ud` Bv IXsxA$E1= 3a~ks].<(ǰYnyfx ȋmZ%q+x}|s aLuRm=cd-%a{˔Ξ0<.__eǗй+tNp𥍛rs&[Z f,qސɐ-GS^cs&7+-T^\`TZFXstlvܭ+fg_%‘"(i֍3nRa遥Y{z}y6l} x_>LŒ߃CB 2zյͽb5kg\liZu/\;fZb mt,ށ?ֹ =lAgTv!Kd eso@@9f9`ѢP6ҕW 9BR$9Yl0u'c0.$B@<[5d<*t'GF@G>, X _ T3kU376o3s5\mTr99.o?4}sX+ై*\=7Y٬ޥ͛XCr79lb:SjKkZL6"L]U _Xbr/fYyAB僑< i*Ko)]3=wLf.fW:癗hȣi OcK4Ķß+[Zդ6#}{*%==h>.2|ʦ$i[r+8SC?*Z)u:EAdkJ{~j\O!7|lhNҕoh e"?xۦ=}Hg[qݿ7_nЗ~UeHکCY5 ʠ\K 52^r% NHϝ[YF|7+=6{ٯ~j?]K endstream endobj 113 0 obj <>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 118 0 R/BBox[0 0 566 193]/Length 3054>>stream x[K Ib.A|\ŇTŽ8$k&U){{Nm @MZ~ow߅/4h/D _pJ^i rƘL:JdY<2b[}%#~}C]yR즖!^3GF,O#}˭weȲXy:!x5^2bM F, Z,éINz2\y ,a z<=y±1퍢x x Y@߈e8߮;&?Cu4N{d,xxɐ;eY̐e_knjxoȲ@y4 T|l0S2qMWi]ThJ bG-$w %D"چ`^7]1W)j8H#ȯU@Y^,?LJ/N2zPW/_kE_Ĥ5]X7E7[y).sQ4ߗg~%KX9<Ÿ3s‚'gW, E,P"ZİK so/yb$ofR >to2i.g81{j Z-]DL;Ɏ#"~L+(>xmj#$]Z':mcq)I u:!mlsYP+W|9vv88U1.\,/ `(J+0m1<'AwrmZEbEzI(jaί_?+al=A:-hds!^?Y LXEg7a-,xÂ4EnX4X3*MC4q V|A/rg-}_8J9rܸ47*߸MC+iim}RGihf9NCy9Bd򟕆iZ,`,0gc\y`5}BQyk! i S\j9LU3_8U=0Tzlsaσ˹r*<F ؎L)Nn ^h!k\pP RzG9a+'B`y97SBWL!-B3\%daTIU{!=t2)pҡH@u2:zʭijTE a =mB{u s,{Yɭ!'R i&!u>Ҭ }(&߈.8I&!io(Hs/es b]\Cr&)Q3%d"r>Z9Y)QX7|p7s bEY%aye(mB%JtkZ<)UˏNWD9j_Pv&WCrd$UъI"VU(H*!qx)V;&I-SlT`;WY@RԄ}QgO[e߲(,I#((ѽ$li}(r4ǽ&35M3Ln(OF˛uH@AFho 3RV;a+Փr ]7wAgg?pT/b9 u*eѴuH,orHհ<,@bm>TWlWC\%P+KMވzYC)\r:hɽLqP:[\kZI/[(¡RrS|Rr+'wPkx$F\̈bBym8)뢳|e9^,b=k9 xaʶ R\+Rb]4H1GI [UTun2fUl;6Uk BbJ\4E#c[ Xh,AM;xVV0Sj+]vB+nK͐$*ܒQSij2l]#ŗeVēy 2/ (l"h"W&TfR OC--mس& ՉڬȢ4s͚yMb cB:-_\OÃN ǰ\nE˞@bS@vbs(IAɡwf pYHnK& q%6r’N)؈}=XɞauNک4RD0sm%H :@ $dᒦ+rSvBŪ$i({j9a+$Q]Hd#%ěSX0 L*e'>7$ j~.5]SxNVfv1 VZ S7#׌5Bv:Jea,rN묆 > endstream endobj 119 0 obj <>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 130 0 R/BBox[0 0 591 439]/Length 3479>>stream x[K# W/MIɼc9!U*;Mqffkfx54x| w5o_btFH q9A#Fu]0!5TA)ctH1&]M /xtYcg,״%Xf.2\vI&lXfbJe6I.ip%Xeb9I24;Py#8!`EE9O1ۖ}o( B?B@~Oe D%@ǟǏp"Ќ ٥:BP &Tdm(dB0B]T6e'6t5 *D>v‹1Ɔ ocCH4SC yb;C6PO,5DtY7ߙm$\edf$n43$_쩿E|ygۛZțshA2`=UdxJݣ#$C=zxQs7#\ kOCH腳t(6Iԟ>XQxdo/o2!$X}clgJ"T38t!PZ*:;ml- $Q}Ծ=, 쌋Vv!*34.w g jiո>C6JNt+\Z) Hx,ċgt˝#ur^[Ř,5ys;~4$'NXyt6#̍b9x)qL*{dxqJzT"F7{ 6B%LQ~!#G^"c^"VVlPNHCd ژwD| { fKPHCyQe!WA/ԗ"瞓 C,ӊG`L \x* rW<)%Ry6&K, ya)Ck@aM,3~j&IM$[Mk2Uqڎ* Am* &*MlS졷^ϭ#c#" E=@?!Hr qX"MT*r]UD= w@jsց1T4p.krwriCd$ uw85tpzM֩zL9k|e8d->R&1~p:, G[hv|uG- -cEc LL+%uz$fR@ TDWg d(q(_z p-BB &fywE*}YrkK.@&=xTC[\vI4L<:iedb=]LGFPB.iOF*=!db=&\5fP`z'5GJ/Z))뉽R* U?Dki^4qa)S\AqF14;Yꑏ4^v5‚۟\%*!εz*1Dt$4t@x Z7uXs* @zѺw$;fmM^fsvf$Eli%3/M~@^f[`Xy85#X}?8 @#gH.{>e%8?22ZéD VWvN|{Mo)B,!/;Q JP|ٿ[.΀͔(XDv\]/({'8}  ?W9a Iz`bK6CZoi'Ý9#9]@JUDK26]e\$i#&&o\M$4\ 3'62-mH6HE > !8c/L$T$** =F$ IT *(3.jM*Aj&_rآaJ(dD!6% 'sIPcẳ1s)7ׁw.ٮv$lMsv-mHND6̌z,1WG#F9J1Zz0b#F%ˈ 9":aF$ j[wb1r>`3,qA\}\$.:%z/)Y&@8'}#{^t1` >tQ,flNЕz{= 0kT#=nu4=~( w?,Nn߆њ.% I$=ĸ.f‰2A(̈R!?ưս53w3I_$(*I4x.(]B]i\ŭXMN.%1Lëw'Ze)Tz1z>ok ˣ7Ak੏\&"wN]R'7̐% f:V֜tf`S+ L!82^X/P-UoB97Z8KFPC֫.+?8(kS1Q8{x9tQ'vIhXWyY(pXt*)+_45ܪ>⇊Qᒄ W+@MVQ/@q)aj2 =n gL"4$98JZ[__S6PrhunC4I3I0N"?XY T6V+(j%$._=愖z$fmr215`%l%n$c\/C@\yX ׋̩^qsUK|~1ޒMڤ F,{`S꽥E@R0tԑo:0SH<<7UF1F9Nh$F#h:tAbffuBԌ&5 H=E~7>[qRѤD%Y@o&~14)G'M1R9z)/pXm䧗)Lw!Cһ)W_qCH|>Ξ'ʕ_K% pÅTq3 R2D2'Ifb{B8WLa}!Xi3!9ztdc<$4 KV~ endstream endobj 131 0 obj <>/XObject<>>> endobj 59 0 obj <>/XObject<>>>/Parent 39 0 R/MediaBox[0 0 612 792]>> endobj 132 0 obj <>stream xڅZIϯБ]բIk|=3vPTL6픪Da>~}cHy2-?  %""59|>~6}Yz(*PUaQd\Uaÿe|<, ?<U?vIw,M'al8qۘ~z?B2 Ǹ ˨:&Y}xl an g? AMݺL,N7=NҌ y8Rߠ_Q1>4 8<$YbC4 ퟱbm"Ƀ n'MÕyS(݉=uWez^o+`(13_(pnxnkZ\7`gtͤBlLsͯ@ܥQ|sJ{/rfSQ?D]!wD"$ WZz 1<n'3O{ da?([ϗuo`7ҨPUM-!ςG.s%߶7g,z6ܮʆ%jM 4a>mWq(G4-Ӭi3ZBzUӮVŌ}23 Y0gT.;:QIiZrG].k8iRv 5% f5_vcnfm,\I]]]R2j׊Q0vCC8Lr CQiQrΏ\q4 BԸ$LWbf28;/)LKYFv*(*9] PEVIjzmzQKb-l0EO"R e0P ]xDYp~4lxl"?D01{k ۅIK Pp 둃YKZޡ)Ps@TEB;-02qC~zefha, EF.O !?u8G F6ye ] M((@_P 't*nixiWM F=X nhո`x6qLVmVpv[1]}`Kjt3]$sr&sN9RwۼPۼ(,6jla%uZHi(dа*cuwtU 7&h"e%s(Z;'zYcnm 4= n1:ܣ(͎du8'I,&KWPϸUu-S&qW* N'S)~ vFh̜E̮2ݚo(G rƚEKdSfk$ 7|O>=jǪIJc69v1X `$QQO)!>9rc&w10j=.,u쌈(#:;vuRT%SΜ:5I)$;%19hݰyّTO%#8 )ˮWeޣ2YOZG:;Q`2sôrRc.D5Zsʼn›} m R3eeNz/ּHCPP@Ĝt2Tv /% EJcPs}p"wv ՛ס֘m| ƶH[J7,D'Velg#*t!Ml=El̈́=.%`rj$XΒfOZ1r+4{M\|3Q `54%@G{G ʦ: ˀ$5;cEn-`e sCXn  Ӹ)ijeyldUdGҲo0hv\yXz/XJ,U&Irw I+ ;SvFȧƑ!ȗ"yJm]] #ִP㸃UfC@ǒd E4G0ܯLjem΂Bw81cm  {^)Le6h qkي(|t7W*؃4 o29%MѢZ~cÉ6( LjXMޯ1bT\4_A3zd~9yౣtAI#,JY>tXKHBݠ9/fa/<$Vi#G@{KQ>*TJEmFYB>9,v!d-\yj/5#9/fdi$u63c{ԕ؎7=^@ L2s@.=lGҷ«Ų7cOB~1 h;͡ shf~],i6љ}n$EQ)漊twBYj X}r3qGF4UH, /51 jE.p]Ȅ36|7i${LXb?{F,f/Y fle\D@a$£ {<0L<kES܋D>j3|Y耄{@6LcN6'9F{D7'Hٕ%,fZ=ÜGK?bMg=9pHx&{ fm cu8CZ]Y\ w毦辝|gаkh {&KB # €dGiWC~`] VF­|ўӫJqoAןoĽbʆK@+1OCPX:b8/ЪaP[WK!#$\&E %MO3^ul'=؏Co]=='/L{t0$FsjbL+xj#lH.ne|UHL~`iw}I>hhb]k/+.yBa$B;qܿL*xyPٝ&'ZW=yz`%RFd{zLJ7!Ѝgh$4}ME_apT Tރ2 1<ö( `@xYrxƃ,av706NFמF=AJ1|g+WqP8HэRR[i'ͦ6 =. pZ4Wr/j vQף %4+e |)Yٻo endstream endobj 86 0 obj <> endobj 83 0 obj [/CalRGB<>] endobj 82 0 obj [/ICCBased 133 0 R] endobj 84 0 obj <> endobj 85 0 obj <> endobj 133 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 134 0 obj <> endobj 135 0 obj <>stream x]An0E7IHh6&VU 8fX B}|.xHX\?uqPYo}.= 0>5MU}zMϤگZ4-4kO;M)ʷVGRiv\GihxF!PoxkL7B, nMwBmઍ^!lxozS5Bɴ*L{!0pFd̻^tC5y׼Y#[9T.eᒹD[P?ɺ~ e endstream endobj 136 0 obj <>stream xz{|՝93zf$a=g$YCǑ,Kk1"8Iv! x%h$dނLCpmMao?]n޶(m-}Hrgi7s aM  b Prx#rnPGC'׎_w}ʜDHu;nP7@Ǐn?vnDNP~ w-N(ޱsK?wPĮ[v ǗC߹kw.i[iObQ<)>FgGef<&-E˷Z褳>j'MдЏ,A z=gz h'H QGףatz@j ZGP7z}vhf6"C[vt+zXDܵ >E ϡ//&ԋAE:3fÓ>~_DGWcEl?<nyIWTܼ9lL4cꥺh2z.gnY =u;jhZR]LWiAcm4oPa z.,yyi%@Ἢ#HSwakpՀ<4bCѵ,9Gż -bvT 8P;aټ9_fc"NMX5xy\x*,R.F k*ux;ԡmY@fGA0#ި`v^ګ\8zg3/ݷz(N}x88'[F-"dWkxj+(vM O,N\t_xv:0frQ Z=q͹RuTERXz@ Ej` 8 z-Ѕ+Q$Pȸh[j "<4#kX5XH" a3[걭#=AID- /}9^mcVϗu n&W7K(],%]! H^98nɉ<ޚ`ߪԒZ+%z*Q<ӱv%_>}:o_5-\5x8zt r3h]oi1Jn/NΠNiEMн>*ٱN lBCU{0'NS[.qtd+H^cTN+H:/K\ 3ۋS`]탅L; AfA%g(KO [ʋ|5< \ ew妦 (Ԕ{ QϠO5`0(G ]A$u½7KvIG~!驋օ6SpЖrt!*!3HF!z r9 bȶ蟱#mD ,F]タׅ3^^ͼWwI2)˼p/piWʜ?T)8/ `o}4Z=k0^8 '騺'nךn9Vp0>11}}Q? >k6L0pt\qp;I$ @mLu{ZQr7,#?c8͡ GsL $Fs7hϿ !pХ%uu;޵Lٽ'ٴk\ݻv* EIF.^,=}=d+CVTY@ξqf9QO(_A7h~yd~Y`$4xxICq ICs?gZÅ7ǡXэRv4۹[7 E)כ v:GC2a41)X&cyА9.E,\\ C#jh7TjYU@8d* _Ḹ=^5ݿ.6ֶ۫eZc-77H}+l8Rۅn9\#K[kpsԳU"e *Z}Hh0ܥVYjbTo42J%6,feRPC׊ZF;Y_V4k z=3 ' 0@-d=8֤J EDp iTd(cIKtڜNC>կکگbr"[ՇٟqsoƊg'xya.=#Y߳o"=DYeW`l`AONMūrJNf.&́pI( e4Tj@:=@dץ}T?!3 ,jE}t<-)Ac7hjV|Ulf25%a(2Gax,;iKa&,kdOyP;` {T[g=Gtv1x Y|_NFUd*Oc 0MXa'x3+J6;xQ=/ F\w"Û"C|,n&YPIHȄ >X{$OJ8x1NꮽrȨjolݢ\]QkoG]?_IŤډko*k ,dڵŸJ f`SwnSou!y͕YSzfYWLL-<6!Z> W0~(TD8Aɕ#"C!Abi% jZTWoVvǒTXuaC*\. &Pz2\R(e u 3O>hkm~z쫧m匥~Cum#RrO]gVto87%T.ت'n>MjnYPpe@EKF:cMTvP;mۻ\=;Ord498Uvt<#jO6ْ>Dg;I')Z~B~'ދ@ĻXLFg3l‑VSsБF"}Sւ}_\b1! DL$A$LT_3Dazly iX슶Wg'=N$sz| k*+ZU"b8x}niZיt xCMRfV:%lPL65TgN XAT&E 4t,eJ6\ܬ0lWT -7 &pLsM [yE"ziE.>vnvj\TGǗi4hRRh$?GM BD E\K'%gŞl=wnjJ/oN?>]}rr1;z;~n4Dgsh}䑖5;D驇(%1"' FDy+uUdGԃxf/kexLeͼ9Ň].tl ,~xpАأ'@i"An{ॱrk(4\P)UW*44S-'d_@R_.{YPe*eK_mr{5}}x__<(T fDef즋Z=N[Ր\`8Љ| Q-(Q!в .yNxða *waPe!blZp%ĵxIQoP@N oN] rfP$)BRx-St5D 1 9rXC4G\k 3Dz h-ĕnLGww}e$3/3om.a+s>ۉ\M !M4#T9[ifB{  /_Փ/)KQ7,UMU2 ЍCD/4+տ}Ap4=i]&q3J%(抌`e`?5t "GhǶ<Fh:D+&w]Nq|^0ܷw%w=#Q7Hp87j]n :qu|U٤xxWd &gPB#P!m!ğhhbkyBc/mh{ XH/V@ vA! A@G@ߟ $@"k!L-H[vna:c!ewD\ 1 |0Ƒf)h^.BDTkBⴅXUG~ J"ܽl}RWu| ՎR[E ;K #ǖO*23k=EGoq@ZmO_+u0ĸOxV.^L\'T?^VN2K<,idmwbF9H8)OOYS$/"MQ$N 8UR2 @,V՜ ) taȜ,ȒcѲc ?l%'D!A h-%yTәKyQKiI"K b)hGF+(I^qǂבnN Z2=&eyVP_n;Op$D;@/#XԵ+)3 c=# 8WWW1> ;`ɒy< dN7hP *K(/I7z Rք1^ʑ.ON˳nD{Jl+'> @+nq]Wvwv^hj}CÏrJNE/w$1'q(DEB]4.uQ0^]c7pɊaXMGvݭbqK5XT ! +|%uw w)OV'+^sk)xm[b-e @m/٩d0UtfH4 Ķ|`f,CTY&((3 bb6])w:wS-y&49|r&MÃW\ܰrhX[zR춮ӟBgvE{zwt0*.P8}):oɴ \yϭǶx0V;ds 6Z4-W1zEuYu^!Pz" V{PiC|Muw$ŒJd1 a'DV ^ ̏g]UڧjBwEp::y-aGںz 6vyǓ6r&ڲaUr#d1->[‹'W7նجA-EbqJ-rdR,I [u帏%&/{#hO4 56O=nJ-Au)FIHVSx|6"Dq7Rf.K=^a0qsK{cknӧNa;Z3/7:Gw֛q|Z^VM;1myEMTVYϪZSDzvYkVFiH|KD+(i (}%AV= 4 7aA%5`LC?t'$~#=#0b#Og$wB4(ƥ }@6ݡo07 {!%pGo'E- \_mkc T-kW4p/7ܑ\uW=NKw_5M:?Zzr8lr*I ԛjS^UJM3Gvيe'B@Z:ĉCLF`?6=dQJ0HgD$l Ξ|g2K0f6JNX.=E.nH2RÐKuk lQݨi [_0n|:S]rZmY-+9a< ehNcTWzѯeVjZ3_L11YkHZX#:_IHT6#.9ȑ:g+5s5 X7\W1$ jq0Pm,nqq&~f%-!n>ZwwW*0Ƒu(:}PzŸ5M2?y?MQޅ[Wfh5رB7T.Jm *8lkB}=.RA==t|ξDen.ZO ''[]H,U s J+JR4׫:VK.WL;ۤHg(֬uεu닭t9 p8uun"(w/%>fshTS-V ;$ d_HdQ $X΂Mkfy,!LX.Gw3@U5d)H B,%mZBF1R2 j.^sMP9ձKl5hCJ&++Woڮٴaƶ/fu>ݫ[險;+,kLZ1&MgsdE6D ŋq f*=ae-յSi:.?j5p2XKq * vRrf1ꭟ PI uHH 2{#RB8ij 5"=*a_ P7]76RA0R~'_G/~a^@{ I@*4h&QE=웨DfvfbVAKqh92/"ATxQA;HV4f#.<ڬ?"ח .Z5m<U- c endstream endobj 118 0 obj <> endobj 115 0 obj [/CalRGB<>] endobj 114 0 obj [/ICCBased 137 0 R] endobj 116 0 obj <> endobj 117 0 obj <> endobj 137 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 138 0 obj <> endobj 139 0 obj <>stream x]An0E7IHh6&VU 8fX B}|.xHX\?uqPYo}.= 0>5MU}zMϤگZ4-4kO;M)ʷVGRiv\GihxF!PoxkL7B, nMwBmઍ^!lxozS5Bɴ*L{!0pFd̻^tC5y׼Y#[9T.eᒹD[P?ɺ~ e endstream endobj 140 0 obj <>stream xz{|՝93zf$a=g$YCǑ,Kk1"8Iv! x%h$dނLCpmMao?]n޶(m-}Hrgi7s aM  b Prx#rnPGC'׎_w}ʜDHu;nP7@Ǐn?vnDNP~ w-N(ޱsK?wPĮ[v ǗC߹kw.i[iObQ<)>FgGef<&-E˷Z褳>j'MдЏ,A z=gz h'H QGףatz@j ZGP7z}vhf6"C[vt+zXDܵ >E ϡ//&ԋAE:3fÓ>~_DGWcEl?<nyIWTܼ9lL4cꥺh2z.gnY =u;jhZR]LWiAcm4oPa z.,yyi%@Ἢ#HSwakpՀ<4bCѵ,9Gż -bvT 8P;aټ9_fc"NMX5xy\x*,R.F k*ux;ԡmY@fGA0#ި`v^ګ\8zg3/ݷz(N}x88'[F-"dWkxj+(vM O,N\t_xv:0frQ Z=q͹RuTERXz@ Ej` 8 z-Ѕ+Q$Pȸh[j "<4#kX5XH" a3[걭#=AID- /}9^mcVϗu n&W7K(],%]! H^98nɉ<ޚ`ߪԒZ+%z*Q<ӱv%_>}:o_5-\5x8zt r3h]oi1Jn/NΠNiEMн>*ٱN lBCU{0'NS[.qtd+H^cTN+H:/K\ 3ۋS`]탅L; AfA%g(KO [ʋ|5< \ ew妦 (Ԕ{ QϠO5`0(G ]A$u½7KvIG~!驋օ6SpЖrt!*!3HF!z r9 bȶ蟱#mD ,F]タׅ3^^ͼWwI2)˼p/piWʜ?T)8/ `o}4Z=k0^8 '騺'nךn9Vp0>11}}Q? >k6L0pt\qp;I$ @mLu{ZQr7,#?c8͡ GsL $Fs7hϿ !pХ%uu;޵Lٽ'ٴk\ݻv* EIF.^,=}=d+CVTY@ξqf9QO(_A7h~yd~Y`$4xxICq ICs?gZÅ7ǡXэRv4۹[7 E)כ v:GC2a41)X&cyА9.E,\\ C#jh7TjYU@8d* _Ḹ=^5ݿ.6ֶ۫eZc-77H}+l8Rۅn9\#K[kpsԳU"e *Z}Hh0ܥVYjbTo42J%6,feRPC׊ZF;Y_V4k z=3 ' 0@-d=8֤J EDp iTd(cIKtڜNC>կکگbr"[ՇٟqsoƊg'xya.=#Y߳o"=DYeW`l`AONMūrJNf.&́pI( e4Tj@:=@dץ}T?!3 ,jE}t<-)Ac7hjV|Ulf25%a(2Gax,;iKa&,kdOyP;` {T[g=Gtv1x Y|_NFUd*Oc 0MXa'x3+J6;xQ=/ F\w"Û"C|,n&YPIHȄ >X{$OJ8x1NꮽrȨjolݢ\]QkoG]?_IŤډko*k ,dڵŸJ f`SwnSou!y͕YSzfYWLL-<6!Z> W0~(TD8Aɕ#"C!Abi% jZTWoVvǒTXuaC*\. &Pz2\R(e u 3O>hkm~z쫧m匥~Cum#RrO]gVto87%T.ت'n>MjnYPpe@EKF:cMTvP;mۻ\=;Ord498Uvt<#jO6ْ>Dg;I')Z~B~'ދ@ĻXLFg3l‑VSsБF"}Sւ}_\b1! DL$A$LT_3Dazly iX슶Wg'=N$sz| k*+ZU"b8x}niZיt xCMRfV:%lPL65TgN XAT&E 4t,eJ6\ܬ0lWT -7 &pLsM [yE"ziE.>vnvj\TGǗi4hRRh$?GM BD E\K'%gŞl=wnjJ/oN?>]}rr1;z;~n4Dgsh}䑖5;D驇(%1"' FDy+uUdGԃxf/kexLeͼ9Ň].tl ,~xpАأ'@i"An{ॱrk(4\P)UW*44S-'d_@R_.{YPe*eK_mr{5}}x__<(T fDef즋Z=N[Ր\`8Љ| Q-(Q!в .yNxða *waPe!blZp%ĵxIQoP@N oN] rfP$)BRx-St5D 1 9rXC4G\k 3Dz h-ĕnLGww}e$3/3om.a+s>ۉ\M !M4#T9[ifB{  /_Փ/)KQ7,UMU2 ЍCD/4+տ}Ap4=i]&q3J%(抌`e`?5t "GhǶ<Fh:D+&w]Nq|^0ܷw%w=#Q7Hp87j]n :qu|U٤xxWd &gPB#P!m!ğhhbkyBc/mh{ XH/V@ vA! A@G@ߟ $@"k!L-H[vna:c!ewD\ 1 |0Ƒf)h^.BDTkBⴅXUG~ J"ܽl}RWu| ՎR[E ;K #ǖO*23k=EGoq@ZmO_+u0ĸOxV.^L\'T?^VN2K<,idmwbF9H8)OOYS$/"MQ$N 8UR2 @,V՜ ) taȜ,ȒcѲc ?l%'D!A h-%yTәKyQKiI"K b)hGF+(I^qǂבnN Z2=&eyVP_n;Op$D;@/#XԵ+)3 c=# 8WWW1> ;`ɒy< dN7hP *K(/I7z Rք1^ʑ.ON˳nD{Jl+'> @+nq]Wvwv^hj}CÏrJNE/w$1'q(DEB]4.uQ0^]c7pɊaXMGvݭbqK5XT ! +|%uw w)OV'+^sk)xm[b-e @m/٩d0UtfH4 Ķ|`f,CTY&((3 bb6])w:wS-y&49|r&MÃW\ܰrhX[zR춮ӟBgvE{zwt0*.P8}):oɴ \yϭǶx0V;ds 6Z4-W1zEuYu^!Pz" V{PiC|Muw$ŒJd1 a'DV ^ ̏g]UڧjBwEp::y-aGںz 6vyǓ6r&ڲaUr#d1->[‹'W7նجA-EbqJ-rdR,I [u帏%&/{#hO4 56O=nJ-Au)FIHVSx|6"Dq7Rf.K=^a0qsK{cknӧNa;Z3/7:Gw֛q|Z^VM;1myEMTVYϪZSDzvYkVFiH|KD+(i (}%AV= 4 7aA%5`LC?t'$~#=#0b#Og$wB4(ƥ }@6ݡo07 {!%pGo'E- \_mkc T-kW4p/7ܑ\uW=NKw_5M:?Zzr8lr*I ԛjS^UJM3Gvيe'B@Z:ĉCLF`?6=dQJ0HgD$l Ξ|g2K0f6JNX.=E.nH2RÐKuk lQݨi [_0n|:S]rZmY-+9a< ehNcTWzѯeVjZ3_L11YkHZX#:_IHT6#.9ȑ:g+5s5 X7\W1$ jq0Pm,nqq&~f%-!n>ZwwW*0Ƒu(:}PzŸ5M2?y?MQޅ[Wfh5رB7T.Jm *8lkB}=.RA==t|ξDen.ZO ''[]H,U s J+JR4׫:VK.WL;ۤHg(֬uεu닭t9 p8uun"(w/%>fshTS-V ;$ d_HdQ $X΂Mkfy,!LX.Gw3@U5d)H B,%mZBF1R2 j.^sMP9ձKl5hCJ&++Woڮٴaƶ/fu>ݫ[險;+,kLZ1&MgsdE6D ŋq f*=ae-յSi:.?j5p2XKq * vRrf1ꭟ PI uHH 2{#RB8ij 5"=*a_ P7]76RA0R~'_G/~a^@{ I@*4h&QE=웨DfvfbVAKqh92/"ATxQA;HV4f#.<ڬ?"ח .Z5m<U- c endstream endobj 130 0 obj <> endobj 120 0 obj [/ICCBased 141 0 R] endobj 122 0 obj [/CalRGB<>] endobj 121 0 obj [/ICCBased 142 0 R] endobj 123 0 obj <> endobj 143 0 obj <> endobj 124 0 obj <> endobj 144 0 obj <> endobj 129 0 obj <> endobj 128 0 obj <> endobj 147 0 obj <> endobj 127 0 obj <> endobj 126 0 obj <> endobj 150 0 obj <> endobj 125 0 obj <> endobj 141 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 142 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurvzzzz'nn///bbG`V__Kww?33(3 K  c  z   " g 6  [NPfD~8":!i5PM%= UI=1)& A!X"p #!$ "%!#&"$'$%)Q%6''*&+((+'i)v-((*.w)+/*--13+..2K-S/3.G05/2261$38N255946;5J8=69>7:?9@<BBH)@D$J7B4EKCG)ME.HODFJJPHlL1RIMTKOVMJQQXOS> endobj 146 0 obj <>stream x]An0D7`'R7&VU c"DȢ!颋A6o/OeVS~,Sʫ釱[}z,)6_j i}:>-EyzϜ ~OMISsLy5GkR玶.+uN`kmZ'Ҡ tv'Jm#Ҥ6tv/=6¶ˣPHkPH BJ^8DSX(Zq[#R(hFhFQ-86jzhبiբZqdud'dP*@Oȝ @OZ!==!2ԼƤDzqqt1<ͺ@/ endstream endobj 148 0 obj <> endobj 149 0 obj <>stream x]An0E7`flhUE "dwtCzg|fʏu_4}~)UmnRaYH߇7~&̑xZSsWĘuEڦ%њVum+ՆW=-у' ʪGrsQ:FU2ITYBN*AU8$jIDm*iy$C"qH=NRWڀ uao؋ce^_U| endstream endobj 151 0 obj <>stream x|y|T9ٗ{g2v̝%%7I2ܐ 1l DQ@pP-nZ[<]ZkS%33!jۧNf{o= :JI@>3X價z&+n$uT5 oY\ݰ_GU!ׯ2*nڒΡV[u]%})T-vUv 7z?xê_@SV0½ρ&$ۤ| l39Du?G}~΀{|<~ +pL+$5 \ ,46n~t6p| 4騥ZKe>Xցi(@t_f׀[_rZ lN/gt']N[lT=U ^An^]u",/JX8ŀ_}^tmVKıF^ըUJCSᨣSsDl}qG3ܟ;sud}(Mx((Qf+OjYNlY;lZߏh9aC)+d:mشJZ"uBcH8ԆD|B-nTӏ̈́z8s.N+PE9 *FUQy(#W+/A[8C-0[@epe@1;ZQ)6n;j4qѱemC#nGZk\)>zU^?.{{{GZD4!et()=S+k[e0g׻=-H0Q##-+ŖWN4*#0oQy@ĺ|S~aHOs?&|cfwN-R F(j\#+{!:Y' #_{--gVdUZGGZ8qxG`K?jW:s{ܣwrk@kwO7]*@*KGq}e0/ Fu#>`zsGHHql N2#)8ED/bI6k/Er,%VՂlS-GˈBMDHK18 T(*P( - [L?[[|!EY U{g^|_^ɼʲW6K3;3S?Oç/9~(?x>>L ?0ǹFP8[KS_FG/('hU>?<-3Vm~Tz''kGIۻw/N1z''6< zo!""u=.x˿7Mw 3 ;;jxMfG~pj;Lfoo!m3mqǶBU`6mtޘwt%Aoh{ >s?b7 M ~gn\=n܄O۴9[q)m2޸\ocnF\߄*AnԦa}{܌G!{Ѫ%`yVI1so#Ey7PѨOS| ؜=E2g7)dMT~>%Ӵ>)6R 3+/`8*)gS/d?̏Ξ͞ AbvIv0#G=>2 oRP |~t |iACOBFtȐ[K[r PTV!NE*@8ٳ?:M0Md} <½87ue[UD0 {]韡ihZCVMܝK)ܦ^Ot($ i.r,dtЍ]?3X_,w}e@8R ++ 0s*ifh93̍$31$gC?L?>w}悎9|ӏhI6>Tf?y,B݁ԣ= "봇t:ࢾNׇopZ8-RjFxCĻ?l0RqK.3kSLƒI }P HcLJB) =^IL%)#,6oJFTD'[Ɵ%:q. mJ..ױ7kfJ1*әJ\Cn/%/& RJL3IsH}}}XLX z Vs]}++b@"bD2`Ngu;6n򪬙dU}uK)VVyϾhp,} nN%hDo)M7Fc{ `<[C+;ibX](Q78cU3Y+a`zgu"UUCkCFc=$K h#[61Z,,Flݠ릺n`TGv PZYйfM+ qqwlAdFGMXMMyY,XKH|Aby !|A>ᩬd_zzZYe}ƪTB\!/5;r&SP,i55GD#VoOU6g#_*.Hwօ5Ko]Y.mer ]0pT]JB+bq2.u*;)w໘.bw Dc%wCp8=56)r2.#EEt4bAAW e'8JeجpYHQ7C54:~,1[i}.KGE|YrݭL1\$X^3cXqC#v*JcjV ⿫X#E~ ~Rt: X졡=W$?Vq_dyԬLH跫:"U}uwouQD>NO'^d jsUelV9kZE7;aƋOMkYW,v;e& XdX&|H,b'r]#@Dx&PfQZPOzD$z2F.S/!N8Gj5;x]TfU\Ť:27ȭpyѯ_;6vV޴8)vwz}7\-_ĵfjGWVO/t9 ԋl^li[:pIwE[(aVNG?rSQŔ"KS iVFpN{:fd,s|9_;11 ?`3VM . o~r c2\3u`MO1J0V +tE\cCӱgQ3e&''0ηI">q^7[epe^5+LE`!*al.!Mp+b qAp N= 9ek j[ .$*݁k,zٔl#+ܢEx..rCg%pKmP+*1Oۘ :䱆a[cy6~r]Tۧo)9ef :&>R W"b4jFn(ѨZ*DŽWz0Dm2mR6: !sRfN"3'/a,+'9LSLA]Èu4hpVsE@O"a"/jʋwq&lɄɋ+(8I)͞MΠs#`k=ĉ.5 }ϳO5*ssbvocìDg%NX ۊT9]o*(b2[QY 9G`:ӺJ[ľJMf8\"[Jc8#MK) td+ N !}k!Imf+r9*+佐Ђ-Ҙח[=_tѢ5-ۮ?Nդӏ_k#7\MwWgg[?arh}[]qO^|}/%=ݾE=+V"-Ft9_.F+ JT*;Fa. 7SXjlc~ceEegڟI^1Ŝj#Hk lW6%mJfztY'W?bq,6ܖPM_6_aݫdQ]]+*=)ڐ0@6tT֔*ny=>:4q>~J^m3EљoiR&1L:҅Q9h k fT\$ nńރp?mq@k%x`ɩJ_%%epuu6 T ĝ2Jfٙt 2i7H@B>=6Vt劵B^X6Z$dDb!^w6L<j٠`xGQ3!nvڨRfo93{!hy~9}>̢ٷޟ`M팴4}臐MFrX'S)]ƓgQK vebe8#9:j83' ;c!adD*ɔt,+QvzxA$T;9,X|1q,|2Y .PSW2%(`!Ɔ~dYAJJ޹|G<.`m lIή%xkI:čq#ȗ͘ !iPd -Y$YѬln] JM2 V'06t`'iFX-~$. d s/`sxRw"까gZBܖn @X:24ɸ 7d<Ϋz2$># )"/ۧz.Lŀ=xyֵlv|IpYYh/{:Ω_ ńxmGZ͸ל%h4OS,7p=w: Z;ow]us+KN @[A\<As^\S2Cѩ˔]u ~fӠFh?EaFApL&Nh8;lوؾgR3NGwCjHtih4O$@dW< d*} I'Ch8oL&7#VޕYG΍w͛15I4U읾ڦXiSr:4 .+4MŻxh,n5d 49X\LFgdcNF H Fꕅ)#p<pI ?cܟeC>zb1clS}ygxq.Fˡ+N$& )3(2*Fr^Xluty^n_zO.Rܗhpl&:nQjA?3vl3Ge?>Msķ)+7/TA;NtQZHؙ$iVeJ>(piMnؽoQݬ /PfIbNR"!^#Ub5_= ~Ո4x V_h[<ębodX c}KsQ._- \]Y |2'R# 榬^yE[EjAmI;͋=Z#㯊wCeOw8zB: 4P6A\?qczQ8A|4 cEd1"D+ćy&'kɑ;yX7 &IC.8Z4̽ K'jt&S]x/' ϨƗ& t)2a&*Ya '9X.9n*b0^[Ե#Hմr%u&fXfU$?YmUx7N4a?%q^q]~Pכ*<憰]5xIE>#P @|) p‧"i'[- JNPb؜ojK!78;|E2uA Ն FGJ7kș]0dEoKL1ÞGY7[zHVkk&yۇ_Mo{YnW~M>aFa.soFٲ"p̘̝O8^-g׮ zEnH w{kf̭mۻWtB vU\9\,X% u Żϻ~Vwh}4Pq51|h d_'ܮǬì//ǯF0fXj[" (AmbjrԖ< < xW Ȋ(awe3WK+6VZÕMSy_ !*gb\D 4kPIad11d N2%JVM!I0z ox"`9It 3 i;Ng>|I7J~Gzi#+{I܅RFcZ-߳ugp#h(Pc 8 '0R.NhP.iapD6:* *mfdV'1;fCK9/>X>.ewnxۡ'z vN4a럏<=i-'_뗡eO}m@pY[kN[sӋ[~?qcirGwl3ӎ4cp ošxVZ_KRm @?UB2 ]sbM87f4L{Z s邊"lrN+*v2'#k׀zIڄd*oa*om"o{ (&=2cMx1/}\1ҔU?|wt+9rӑ%7;+sfg-y`}q,xkJzU62؝eL1&J&Wm7P]s.*ޘ?:ߢ@)SRT1اҠ5k/"1rȯl2$YcH#i aXL=6Yw)߃`?cO'q yXc7aǟ+x &Sb{ފ*&*RdQՉaɰ@} 0%7&׋ fc;Ôcxh2;%o8Da M3kcHb.sM^w4j>͸Ycw{q2XN,a{*Y(ɏ]S̤S=1NM ڊYX 2$r Ǖd3 4ڀPNc|wL;K̹ub:FT¥¾Ѧ֌ԩmҷX;eڹeҀvQyf--fOkP*d%tZ҂?3F(EkhN٩GXcp-:3VY) @RJnHRIǂ8y@Xk R  :%k}rʟϠq*9N@_o/~뮤^L QNM6UXseE)8I#+dobdswǽinQV) *`H:x а?#[ǣog*U)Bqk/S?o\1٣QelzӲHҼt9Wjd Z {l6?B6ou$G:Ν%27DW- =䕮d,YgY$kt+%$)ttfJ ^jgskUeZܰuk.Cqf!l6־ce Cpum"8k S5&,#ؑ#ͫ,5⻶OW56CβZUkutd# )1oHgaKl)HYKm^>}'Rs=9Vl=f>VTx_#Q&6ynl }3c{$m|1Y,}nqW̒H>ѽ[XV&PEFJx{[Aɥ%hY}.}P |4Z+B,u2Exn/±:7V6n -ȫN6M6֛*}DB%B h{e+Z(Çb*xjM2+Ҏ9&3"侸` Ҕ ;+{߇*^JD3q#4[e=z S\R,PV)TQj(!>j=pq:+sۦ/OZFOlw8;g<<@.+f f)Di:ZMfEjuNw{}#+Ca(0'%a*DuD=@$C ؀؁8w D!shJ :3h[5kz |8X7_;-:faPԂ:f'X(@/`T׀2A[Q f0 M0z8"eTQ &1툞 (0ρiLhe?ћA; &.3Z?x?䃟qS-c ܹVU|){qe~2!|EC5` endstream endobj 152 0 obj <>stream xz{|ם93#YOk5#-Y/l¶dȀ C!0I &!@B$&PBit7,I 6n7~>dæiЛ%llo&|ό$m|y|sdJ+S[-_g0[/F%[wP7lVWaٺ#1~wcN#.=O:elفr!ܛ{42殞-96uPn8>?*fE sU<|m=!0gQ#jI.xk-0[`.> KE]!+6 J7P~la% i7"y"q:^q`bp.C=vAx>>B }ˆuC}`"AcЁA| 3ױb=9v?or!H[-|aSS]U(pe~.u9KYmVD zV&UJ\&EAU5?W7}KS@N;Y[fR@M r3)K-PۉޔĆ7dhfI&D析˚ئ~&drDN|(X$\7Ki)$̧M$"f4jerfM S,ST^fc*ԓ I&>g5B8@7cO&EɅ̌8Lxey8Ih#)>eNiPNԣe 3>.Lv>EH0&D75q~_vM9wn obF dbz^>6ט) н2&,AϱrP !$,KbM`lO9YN2Ap%hCuL ,>FhTGD%$ˌ`䫜G\Bd$GG#,MLeho '[bhTP@D qrilcVPx3'DOR'OISϕϝ]ONO9O'=if>Fc{A㎥G_G)[zYZX]NOu貌>r ޺DсK\&Qg^C/vca# XJ(+9C=RWs3Ň7obϤTKs?kJPR]/bы/b"V G?,ҏң<0"xoGFj7gF0E?rk 3(%ϯx K*D` F-VeeR FFS=4U zxhwӤCHB,THer@ @LA8F!'nGŸJũTQрGq,14( \ xs$tW41\ژ z'%`I۞vFMaGt*4M+W&!75>l rT<TOƁ=;v @< ~I]gwL~6OHy4!w013O o7oC!~PWȏeBWDw F T''o YyKPQ| ?8n"#`4YEڪm5VHcژ1f]$+cQJAEeRՠOiIB'0V<+^Q`U$v|UZժtb9խa:1 ({qy #)(Q;CR׫QSr9*ɹĶ`%`Np А5\b:X yy'Г泩ivܮ㬖S]kG)$bQe>_2o(f ;_F}9y gwt:Ow"\ ^g( ;%kV9ț;2CPEK04Goi"֠+*PΡ%n!Mܰj>lUQ5BW F4;۾"#]b!Sv[;0R" xkfKq KU d L'/ i$*];hH9㰑0 OY>K`]cP;h|d1h50G òBr^n;vxH&?QH/:6׉Y8XA z;<"w]cSYG'lo:XwozNkuTOmvHqɅ*r϶-\u69˃LjxK}]*wYK.Lwծw:>±HbGo'PGN&@)jica"*-/໤E 5B@FPuu<; &UjIOsNN \4r&rXUZ@&e h _}zh;{棰S/T]w:y˽wF_q'@AjC?ϼy_u=];XQ3_0!A[BnGꏤ 6$GVⳬ|6^ުj|Vp.DU073ϋ7n+nVEp銆鼭H@q-Pu>Zg9٘#`9*(i=h\?\sPZW66; 4nMj-˲ ^,ӓA=CgJ[TU:sVX̎|m[rR)mٲb.݇<t:I hc r]PZZa?⭏Dآc[[a}+e & r I.YH1& ,"Yv|'n^޾ɟt$\(/doml7<,p-m+k+OR;w_ٺU]n)~][кKC5nzN4]cd?Y ̷^|/Ki⊗4]!n:4VUKDʶB{vUp>-캆GލD82_y* zs5b_ik5מ\͟lpR.p9~%L< 悅`58Z-+JK*f} |ukɢalPC5.^9Ƙ=!Yj/| @\VEF[\w?²+ _za6qR-0oplP0R0USG-K S0Y .yɳ kd˥u=emg3cVop3.aKs`n;{E zCΠ5KG;;U-&-MV{;; h @rVj|[MRda2qwd )/7xKowrѸĘ? a]H0UhSiQ,鉝]w\x (%X"ͤ&["y׾ѱy3𱓂5! 4 L9$Sfe62'xs)iær-cM4Le|`DsZYL;MoA]d匋ƚ-c"2H-+G 9I4as# o>d%Vy.iV5^y/ڇ5G:=-]Ɨ/xXo0+,{`#9[k7uA)Pe*^TPt"c()e>ep(I }hywS] M߈3 :v}8 5 u.xюXg9E%늖-hYfԓfEڻ"J[3-sfSbeC'nyu~kSV BŸ[ƢZ|+>8.=p$;Vv 5 W-]c8\K9R|M?,%n;bJC(\rfZmMg[([2 ug}"|; wwXʺX}E}drksFU;81ɑjrRYLe4 t$p*k%JrD&)TD~ZQFwuЛ4Tf^mWvu;BOlxvXӲXMlX[p eRTW*eJTåb rL)eR1 LJ? 0 hA YH\wy6֢*OPLs(!Pkqų/SmTEn{91[y"?n"<q-xge0{{g1Q, tA䎳2lZ"flX$"L?UP]D݃&_mmb:2tz?>[di2 kCgO.Vy̩7[+~]{.{!mri;pe2GVXJR$KviYJRޭ(GzRqRإxy ZPAV V`}R?z#u`8iNP.֎Tǔ36u. !=W9'%%D#/w6kAi"s+O~?ԣYQwk~Z!PbHA\o>R}'jLn2TEcOcOOETʫ@qF{6/?q{fq\\;1WcIB >*v7,,@R' d?$_tׄD^(Si "1*z_1?W9sjfv>Gu?4A! .$ @ @ PFhjSҿu=ƭVџ-t ܚ !P_p)] ]- L DGQ`׀h%⻁/C0<:sSK |?GHVU;` ׏^|/^u#Q W*6 endstream endobj 153 0 obj <>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 160 0 R/BBox[0 0 533 202]/Length 1075>>stream xVMo7 ϯPg=feREڠ(xoNNN?H}fv1{].Ph({ԗ@[sXC3*J~~߰{?݌fX-0S$!Sxbp> j`Ok{j\B"Kc_KC~.|k6@_ 篁Det7܌L2x6m)B:|Rs6\ "UNj6楊g.㇩i4٨UI}ܽkqqjql8pfHݧmnE!_ u"-~v7iezY\o%b$(MV (vL a+8 ZtZ[_ۓfoN8qpT2^ޞviLѝ ?%3?pru-3;c|&L50z,3@y lMT"9:ӹDάTO[Ɩ(\Y>dRR$Z@ f)`dR)Qp(Ƨ j2RSWZ j Gqw2G |A&".艚HVk `T՗]>7Uը_'G./Ryܴ`!= :G"dCAqeH(BW)>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 173 0 R/BBox[0 0 640 219]/Length 1863>>stream xXK7WЙX4|dwrr!Uj}C~>[R[-udWjJM#~@gɸ_|6 '3`b9gC^9 6lGT zކAѢ9eؐDEH>cl LC !&%h!gs 6r![nل>]{YRa|&O.ߏKkFcY1qټ,*d?J3LdKWCa*iB6y5%$%KA8M!ì7>pؔԅ O/ǚ#["$ , U=x"qO%; SG?:+Ĝ#Ӫ^]5v_~xSXj\Ĕij*P,.^@㔌l.@ǙVݏ E/%*gdƘӓxj%b+J@)fwց\f[$ΪaWON]N.HlƓrE^aXR;vbp)J''jfV!=XxEĩ/f.b%(vH$T2с1}\)((DHr3V}5w@ [,{ yfǹG`MHR*yR)LB*UwiTfQHVrE0Uo4vyΖf@0Ss)'/qJa!Q, t)UOJL⒝ñy+X tpnWJOQJ?w33OI[tقuܭaml{Xg:vζuݷW/蚻j/F5xQN%QOg-nݠYco4/θAuf 0h?xn4*n@OYoP3N3|gL_denI$޵'^;y|@ϡMw*rNuxC9m]K٥e o0o] F|Nh"JҸhmѶٔ(K Jd[ ԡ3F>39R8ev,LXԷKVʛpo0__aмjh1'9ƃ,fXc/S4S}u)By=Γqt, endstream endobj 174 0 obj <>/ProcSet[/PDF/Text]/ExtGState<>/Font<>>>/PTEX.InfoDict 187 0 R/BBox[0 0 558 215]/Length 1674>>stream xXKo7WurRќrHP ҞܴMPHrGrWڕZYN-Zj3o|o%ʧ]\xnᬚ7Ba|xkA[=_W2gYpJb PfB&0W/˧5Ϯ 4s 6x1Ylt˕.G */7&<<ۉB58M#s\XS6+a\ XM|<@q]S`$ x+ɩudۅḌNjͿUԴ-> &:^n&611b 6!;_AtHi\=Gh$5kK& ǽ,HCދ?xi2+' E.F{^Jm\eP ;-7@zZa3[׋կ0XJBLv6GCUs$C;j#Ch}7KY٣wU*GНG4JwyDBJQ @rkZ  e)4{Co_iE,~8OmT}e9GqvOs#LfIփ.֧.!*c $*b;ɖk#uR͊RF1Zm*?Ԅ\91k$;|V3i#dֶ/$OX@t$US- HV[c>vXX0`!Y|˒mvxg9X[@\JH@b$U/s[u_>UhF9ثv@9L; =>Ĥ1uay̼+R1a8վ۔wD+baهNV :`VH(I8xeVN3}a'L_<8(>wQ< 9叅"f(:?.;h6P"Nן"`a'FY:C>\`?|Uٟ y@r U endstream endobj 188 0 obj <>/XObject<>>> endobj 60 0 obj <>/XObject<>>>/Parent 39 0 R/MediaBox[0 0 612 792]>> endobj 189 0 obj <>stream xڥ[YƑ~_GtDB]8vF [Όcc}VYT=(Ҕu[)ꖻ>UZn1ETq0v}RqlsϗDUuh|^v]O@FlEvv@Iv]G}[(߆-Vp긊q|t*tdemO|yȋ걦,n䖷J IG{y,n`qb"APS}o1O$X8\#b*'i;WȵY!Q1K/.*lA&M, j|ER|mT\O p:hTհ,/8P$5M\A8×ϕ@c-X-_^Vh8҅umA\rhz0f6]G8 `&Q7t-+;"Q ka? yzj"]U{(/õ}r㢪,3r}0e#O`&^Tr w{qRd˽t-k_%/vBA)Uf@*XG;wH*)4`Ne-p~wUiJ}S:i~۽]{ Q#s\gZ۹N$o+h< s>cgT+zLyGt'`aD( )'-+Y 2 *ߧy{&߽ß?"6xN%"Z }$Iڀzq~_X'ӃކP6w#%VrXY_DD̂_LLE׸y#l% H6Oj5h.o­*O.xdxd-#" Wҥ<䟰Rb[HMP8T0`F2Gr$śDL½oVr0Y6;o_A:olDZ_[M8Wݜ"#ꖲ˧4-[2e'_ &=TMÍ1~K@K=͆x9ÿea.=L>Z"A`&yc /7 DDV-ڈx_%pA aa?0kCqFޝuL8kV[groc3dSoàAG)߂@BT֐WL &P"q|kCA."N`kOٲȋ :p(.<*}2A6 k-_("pڡl<Q //<_AS%҃;#PdUQ۸"Jg8*L( 7td9_Pd cq[L=al/H<,$##rghR=n~uhPBDHlLt:΍46 l0h 6 "tN^vTMGkli UՕ35UN 1v(-3s{Um /-Ճ.Հ N?5FZԝjȥ:Ը:FC+keOVI4&x⯁bv(9A H2wѧ$vhMͳEw`$<@.Vvx9LغxSL_@aQFw:Nț'pnh*WLHHUK{[#⟽rs( ]—C<0;?sR$~Q>k]^i}8Wz:7wT&9"M0Tg@[*laktn7GCdʭ9Ԉp/ւB 'Yk@R/ab):3muTȚ7X` _Pkw/WK5݁cqa76:!ltL6dy?d"1k:IA^phkfѷhP נ=[pY~4x]R>JFs_tf{1H> )"d0V->B­|PsA+, P%0y󣸯02X|OzJr Yҕ5pKȊHPxuc0ǰn) }Lhx"R-+<edMp)7D)0, @ZK\fUQKZnVQ?=I}\JПb1RDa䱫dPAB+qU<$t +[)l[#tm#TpI&rT4IB!\"ܹ M#řۉxzd@a؝$F<7cһ$"-BJG&Gh%\&by5U{紖9gK3DgPA5o81[UO.B\?qPXC~3lBt3"3%ܡN^5#P,eR4 34?.bY(R@eV106-, ❜Az343cQK;T2?H%Qv~ufu-<^C(j\96‡ʰ)‚劵$ZR[@v:_FO$zjzG3",F'|Dcb,IV]`d, OiDpHD}->%WBߎ$Ԓ ð8g*CFMqO/ܳf)/1N&,o~? =[9iDlJDrmb+ 'f:[I~q,u-`1Za/ԯYAa  =`ҁzj|w+$Bev>0'%PAzN9<#|/V _ZI[k3R q]J|v(ZA瘩qemq}"!Sеe9pS̻>}UIE'uF?-! fG}ma7Si| 9^-6~mV6S13ֿJIL)IӉKPwP e8!2#y`6FX8-Mf$pXw(%n4ܧH-bx7k0f`zd)yw {x){/Yaτ<33[x&u+&M<3*2@cb0nts-VLZ-pg6 \=wISt*o%f`=/nCl\MP>uבtC'+AygFG+L R)"[>]5Le '/}GqI%3ݸug)#l(^_J!YKჅC/%6 gc}9wr0w2_A{ֳ)GnO}u`g%xѡnjy6UNl`QHT N$xl%pF4[ӚY2xb @b#/?kh<:ӀmVBFfVQM*xU!,RI9$$U-Μ[Yt\{7P>_~,sptF>#w3K2=lOYcS\_Lj~XPh(4N|ن]EoE'fqcS͖/n?.Go |o8Uzy]OY Hߏ } ٳpې9)WoBȫUNVnٙC"S}l-6`h(CcTr|&D"[!ˆ9-|<õ\KlsYoMF H endstream endobj 160 0 obj <> endobj 154 0 obj [/ICCBased 190 0 R] endobj 155 0 obj [/CalRGB<>] endobj 156 0 obj <> endobj 159 0 obj <> endobj 158 0 obj <> endobj 157 0 obj <> endobj 190 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 191 0 obj <> endobj 192 0 obj <>stream x]Mn0F7xih6&VU =D,b!޾3I]<njzʏu_qi|_A.S.R~YiW)?MsGY|=Ǒ _)a| j`}WѴf`0je2PL; 7=0N=mӁV@?J t0tdj{&= ClͤY y;k&J{^iVB:jVBUJ{TҞҬukY yۄI<܆b}Jp0b,bR; endstream endobj 193 0 obj <> endobj 194 0 obj <>stream x]1n0 EwB78v\%Cd > endobj 196 0 obj <>stream x]1n0 EwB70-q\%Cl ^Oi\Q%K\[ &9/j= psUiĻ?}?}&\,Ձ1r%iWja@yV vѴe:v T{Ӟ@;Zݛ T`g@c@@3 < t5z 4GZEfahz 虄ee?3vV /-%5۔S_ endstream endobj 197 0 obj <>stream xzy`׹93]FX3Ȓydٖ=-cc0`a 6&Y !āHXI4m/SHr[߾n6 +Iymwf$;6F3|gs~.@ 8hi^ʆpy+P{sWI7v1@=պM%~mo>DHշygӖLG6wݕy зum͌޿e@Go+h@|fbrBlf}jfo:wnX.Bw fpJ CxljHVA-:x4ϣNՈ҈};󊳠P\up\w}0#@=Xzσ+0^>G4߆K1ssao/{$w ,/  p w26|\ykiI0\bϛq96Ŝm2:Tg L* 5mfp8Lrs;璿w$A֯M}}ݔD@uЛ2=膄ͨeH:f"=05`L@54*Pb HU@BO`Qޘu SVB=z&gдٚ.] qMB"ސ`=?I~eӵ !ׅ8h_+c; +>zuE"e5m{Sք= ?QFmjGh92N-nx<mx$ʠbэ(l0>S΍;7vnG {/ACFD{hOWOuz*<@6xz6HݰUkM}ҙ BtwPHt,icR-#ݥ8jjVBK2@v2ӿҕs?cb##1ttM\$320DomiC&g.&b #y -i:4fl BP@ Z4jY[܊pj뭨~8Qo<5KIEm6 Z_G{f{ =s;w=UF]<7ݟմV,aVH+&?G^g?!jViR,ϽLmttdN ҔIy9@tdTzd 'w.]vDsok j:$Xh O™IPsf5^Z$(">XLG{FՃ(x<$O>X"wb8?}!T~(##ԿC`{ץ+~u Ly-]~zQN'9 cOhǟROj߿/7G=4Ga)gG}k@=8Իaz u>5 .4;N}C4u/;0,TYy^ۓU;,ڡSYV[*lQ&VWI>ʜ(RGIRR X"UH ĐWŸZͪj oƱf| XP@XI)uˤ(R-0m DϥՉBä,I YK{B jۋ45AC[maYRDs=H 9D̉?m؆.7\Wއw%0&޿"8L턌Xϒj;MCmu ]g 4fgf a(WF d=?co+EAȡqhrQѨ/v a3zS =OQ%@z:@@:/Ӧ*m1l!wh YT7:ɩ)TvLMI `W"wdAI4&Juh __F㒻SFWtmM^p,>=/ߤ˵-'MLRKr:(oj4X"ؚ'_I0,h;'E l YvD(,]$_grY/-j5p:C@<9@]XN#d@t![5]#R9]7V\- FSIɇ ,0)1 'RzoַqM Y[/T)s+o[#wojhx44R*I~1~\~[X7XQ۾pIp(X9͜WC,p)8#O6khhpO a!1bs>-1csEW&WOr 6FG ՂDC?C.QC@*vE9G=+*ڜfVHDt9+XE$өs۪lK\zQFoZ/.M|BR&p4+?ujzIO| ‚9V=N62[N)c!) P!鐔JO?L&Q!{Z{o{i{mR4xw#ɓ*t;@) 9b,8ܷD3xE1ȵHQ-A˓ Qc\Fwڠ ^?^W,Mi['{OXd<<[ WC$Sw^;]"﮿S[]ŤT 2`?{ #샠ʥevL*~x\ǀ1//fx@k3EiK3@Lv~ qX_QFC eJ6H:.{;zi\tT[v6P+~|{ߥXжӬ߾wۋz>&@lF碁,JʲjuA2=Zis38v/9 |ʋ~dX3٣4oR+;́|GEv^YQrՓ[hqUZ3/h۲^ZW[]k({|-l}3VܗdžxRcu0,n<Ϝ5ml;lRh/J5bx(m񩉧!A_Ã`þy9jjvujA]@EYlok Q8 'Yuta7ZvpϢ QAv`ή(cy< +WV_+ݗ_Q|e<}bz/mkk"OVl-XmwAٙ+vkw=Gpb^ҝ;׉Z_Qg*$L[{sg{h~gALؗ+,9$wd^+!{-a-mvkݦY,\xK\,J#^oQ_)Pʲp[7m|R3n8Ih!ndńx18 Ghf/A@C:#zGIhv,z4|P*lȶLMRl9ʛˡ*"JThpΙSeHz;!DR7?C:ID- C8CJQ>$ KA lڳѹ$-)\ e':ڔc5RwK*xgCwQx3ԪoWHX}= DN$z ϔ9B- y O($%X A -d⎋kFk߈uH&l$2Z N! ҂YN/ WY[O;AYk/;lOsɲJΞnY:>#ϭF}Wy|7gZy%!j B›\\L~"Z-VhLה.2fiZlm^QE܁'hYN+;z|ܘCT=NHl.|szS'?qUHy=Q7t0s5JW!^b7/JYrJI>?ZĿJŚbW1'V"\R&ۨ㹂W*==Žr9`F|:^r %y՟aą?za3HAPu0J cB&mEř{WNY.[mHLC%H;*ticl)cn@(_CX̎QKܳ[}1o!C ֍\(?2daӢJscۢE ~p[gQ(^vK2̲WW,7F$d]c%Eccg4UX j^GCX⮖ }z&QseqcfzwKga yt"W^>Cg䓠t |+]`?V4wvlO\O}λ"$w ȯRȝb1D# J" V N0[ 6·Ι T>bXgEEfSտA.=/솰V|.Vsrr\ئ{Lgf:W~o0gWInΚy C~DbsQ@'Nٖr0:fPe k|J];'o7X3.:]Z+Ǚls~ ?.c]eO(Wa=9#a[7Sܤ3(TP1H6SE KZ *||+GڮY@h2 Rf601P4,P׻[6y_WkM8;  kfEtԉր(~TQi0G x 4#[]Q4mDcka5x>hh棴 :D k??U_me4V&`  endstream endobj 198 0 obj <>stream x{y`Tչwιd;{w&B d$$HX&B@+Z \*+VP$hA k Pj*C}߹3b_͝s~@'06ufQ g\hՂ5~Jr͢I>| ۲dUɮ~fҕ7.Ix6,k]+% qIYVQ_EZzQjx"ZH?\5ucfu5E0v _AL2ө-8B)R%v"5, "n$yXQnwaWʰBeءȏa@qV9FYDbpYx>F:fOB\Xܼ8DOJ#w'?:) 98QyckyʈDl2A!)%c / _RoA|L9\|W@ IBFsHeҡaEڱP<o0s߀7so#F@^H$Lfd=I5n_9]I{Iaos zP}J\*QO-J@*q, =$gH}ONzz }*l,Ǝ >)UJɨ[ 4R|Ơ6͆&\6؋E9!^IUHZ2!m&r3rqr&G/TZB>E"z)zYP( Fa4Lֿ~sqU;0|nກ/ T1)G@ x&Xg1p< |_`@x*Zh|> ,6c-6&|g\a!aŚcW5ڴ?H?u002g;%?ȯhyw`4ZK+è@Z`q!:r֡K-K\QӮgot>)'#A Bj&Pv@Jh/WE*jQ6 _h( sIFȭJzo#}X_jA2ʅ3l D?BJxV(rǵdF- Q+khy6r/l. qUtA!'Guz/"7}y\#l"t q8p=b!{GF˔YjM9hh!3zN!7ӕdDCb8)%tLȂĉ;F)q-KJQ/SʯOzijߧ/SɷhbճbUF+T.+Y2hHA~^nNvV8$Ǩ]aY,fѠi5 [xNK\ O8  DK\B4qE%RƐr(cI%JbV(Շua)ANou&)ާ'mjۂmY RwY'-R}|˺[pn6\jVF6M؊{kgQS?ނuq_ β,OX_aqR(0ᚸ5@M\[שHfhז D̋ËkM|{{6^ƍWXWwĻ]]W658fo[oA&6̔p7zGSc܁[JM[%߯5\1-+!\^ֵEÌ?vP9zkVcXWM һEqc/& +ٓNf˕KcjK%筆8KWBĥE1TAk9t-*G24_Y7Զtp<dR7j(fA Ͷ}䒪`; jQxqjlX <&aiMc9؉wNoL%XXQ)N[l>98rizK5K+Ϲec/[ 3 6J]-)6K_KF 0ur%bi4DžljU^Q+U m-(椄rRRnj W9၅0knWqx@]] saXbӵeP @|&|ed j+04;F6͜xІWM{0mijc F7;*+?DHRAF `h`MK?ʎK`5FzӲJ:ymuadUu; krv0 pVu|7; q,Gc!b!bX{=ӓĭJ^PPv/یW 6UO[.z[m Z '5B w3ajA1:6vbv"&XcTTTT_#$NDNT;KR]=Vw j#kb`bkzJG[l\z wYp CP۫j]RmAZ9d3LA6MRi}^OaWd6AFs`=WWa6.X\q ︌ ux:dlŲi3aXJIX>Tj5b8bT3ÑH;aEw"Uw"rŃ9%"Xca)yx.1Ns- b- ݌YXe=X`Y8"q",Sҁ,*93*ZŦҩL@FKzdkZlʇG09_uBQurXNc Ef"3rsq~JUƢ`aDP:;U861y9yH:OòˑXHU搪!\+-BX,C 򗌱VFOłnȷPnE8R؊e? ;O>>GG%2Qzي=܍|64#:luYGֲG) i-1#-vچWy`!'UV1zby,γ<4ϲ}qeMozV/bB9"jm  jF9j;$;YmYCGAB)p1Vj5$hY5HCB̈Gi?h&&Y pB ;n}wV9+Вo$#afc|oFS7br: ͫƃNZ^?㍝uM@B|Ayҁ@]&yT2ueyfJ3v(ҡ4KU)ѝYug&ߠ4Wp&/> L"k7+@KKK|˼΅tjɩ ee^К ­up=iߏ7InP?{CgRlR}xA]SO  ^O |'^O&&{Zj&9պ-m[3NJ{K`܌9 >4zX5B+CiR)_lk[ƿmoo_wmVcbvh_f5s߼Guʴ}=qpyK2iR H\}=A*N>6qL,K5Ƃwku b>@ h``j coq3|7y#Sl磓SlF'1GyQ<.۳eT;z1 Gqmi y0 =G{]vRX}SݿO[|TӺҺ2) a~z;M7l>/OErvvH92Q2'lP8*AMy`)3AB[BV_i6牻lrX@]eI\%Oˏ#iY/=[e-_}D{D{Z+h} {Y9_Idrș6L>GEQ_Q3 `UᨰW,{FCC7!ץ#SRa){rmHjBRIC&8|4ep(t4'%G8Qf9Kt,55>Oyo̦ %cL[k#=k~ʟ\)7]uxэ?z3Vۦ%Fn ~_mS+g\l|PpihVʷBl833eruɹ칹++|˳ ovu]ogsmlmmw1cHqح>1@|r+hæJ} <A(Ĭ9Ěن7K_=ܧ v>*fY;JIxʰ"pكV" ~ޅp=6p^{Yҹ~{:=''N=Hu1s;g*XV@Ghh4^<4Cyfc,͓dQL b&M')AE0[[lt&d-şhԆ_Ue<+ oJwt7 3) qs u z$1ZCEoXddPêsGz)w^@J|'k߾Ʒ?ޡ׼}ŅU\{gn ؽ?D&@bˍ\t}߇}Y_zAz)-Ϝ:8/07:2xK`K2X׻cӏgj/9=Cn2W-(A>yBR%-xZD\*: um}Odf[&sb0&.9_=D1/i&T .׾YLymib}Mb{O2ngJgvnws?FZ,;fnuns@Ɉѣ̯l!CpAF$ @h8c057+/r^3$a^Ao .ƌHjnY\1[.+[jj|9us\U(^gU >Tɓd<8XZA$- B$$Χ*A#DeLH3 jor.s~+ W趛;$nx^d7PcvŻ7ZsLDBstu4r-Ah:!/*6T|W_ޡ?g%@`%ur16]zzL{LO{=z֦E3%(KR &_.s##M CvSҟƴ41dOf@6ȶey4#-eGTk!r$ Y90ttO6diĶŴW1aW~M<iCZZ;QRWS&@F+5զkAg$M[ѯZPĩ?kk若uqT/A3UCn}%sijZ \p۪_8MujPvz/F(j ^Ku@$SL͛De5: qf3feWBDk*Ѫ;:0,kxZ3-;|. lSl7M(Z#F-ꍢ:P؟vinGzҤH[Җ,Y= 4g7zNu2k;9d O>07(q`j.N(ZKklX^MWC,{Bzb)ySS>y5KqZf&t]5CK[Mss˗P~K]wU>&{*H~8gDllﵦ\ (-) GŸ7_رx #3JR7B9?su&1+\c*h$W_cl63$$s>2nFoǻchO]w ~@Y>!g;=\A Th *ocS*߹n7_yCﲠ9>OՏ>cMW)@Mݖwh^ <_4擨 uSeѕ<߇޽R x,BX}E;ƳFiwmOe˦S&GOt܃dm̥>DleQ3'Ȟ=2+KWIhͲ$nXX7H9ːnKפIפ'gЫs~VMqϫW`֗r o<&cdnI_rʆn|}G"wO؝\l!7Z8RVvH eR%0IE4<壄2"y=&HG0+u ]x 06K0VJYiVeIX^KWbyR_,;AF&^꫌䑼Wx=FTRNG9xU̢Z\[Q_Ԡ).UoyEڍ$Z jujbV T |Md|j|6mgo{?M/|v76jG?{k6X$%k+=Ta:t3 7K^P+`[i Z3ӌy[(߁/銘N'm*[,:6kȝFJ4bIiq[Y%hLf#i**j#3}fbyw^cr2b>stream xz{|՝93k$YeYF[-[-bMlK$v$J$'$ɃgKpHIBJ¦ CH@{Mgߔ@),\>lI&vJ3͙:B!CcF=Ca$}ʪXyHB|>[GoAFnܑR_q7dC[׿~WnCޱBѹ}\i1x[*x>5c(7 {rKdP~ʵB.Q~툩&4&*!w7Ryvi@MG4K4kp+FF?CKPCfD}0Vr,:v>Z_DQz}-EA=ehmF>,1zkxj?AНnԈ@n);Vc*6 +i[2: 4B//_m-C5S0ՌMMѺښH*TY |^[p9yQb/Z-f5uZZTe MaęI"`w:Ɋ\ a?qfkߘT *$vx% sdy 6'&ŷl[3-4hX>-E=Q mC TkcG&qK7NRHfL 剓k[F<BhNcgjakLS,3Bz.5#f!~283q6-– AX$=~"8|Ra>>OD4B;ۡY6psƞ1=120+vz"^'Չ{U:IL&`qVxPE0N9lIo#6HOzih|FMLķ-[Z2btCkѵ'sMH=3+60aݞ}%kx'+~3A} 'P=\/)3aVW22+Gf?z`Ea鉉'&S c&&'Fixj̚ZxȞIܟ̰a'ٝdڛ"P)P,: @ʨɃHAN:{'[Fd4T(t<4%MPɌy,C#Mzf=HXgqzZG$jZ2J⟁ÍlCmS,EiB`͙e ᒐa9ɳFB||bQ -7%z. OL(}jv`ޗhU">z4 q`ҭd.=T79ƧPcqt ||k;hTB @Odq?ѵeOÃ[YC`z`k錘/Cd#I>g" 8l896 /0El'o6*9^dJokQna@\VdD{%'&5fb>Oo4`10$땺])$ c.(?ecB]1I-f4Qe)$?_\lҏsC)uE^`1bSlx{+V[ Zk=m|mkrRRR%^y9\dV#>G#G}a)?uEOsǕ܉>q \O V֝9=HGQ['ץ_yax"%^TiWedmJ#?l]3ku ONէ7qgs#G_q?k4FOD=/{w]96ĽDp?vp?s&B}pMJNGw]=8U@x%4%ܥc7266F5?%y`C8=:2zuV}Yӻ=睆RNo/ql:Pg2r?g+sV3CsXVkUjVPjiFErR<&C3WK8bRFޤ JnPr^Fpԍ[3f խH{J2@wFջ.65C30(֮6=.v6O29* 82[22#d&L#$ gw#ɷ3o -E;+P3.Y斾V{';go at#=owvnRC(;jOwq]Q`1Zn?S "W蕷_*i@ac2# :~ }4'sRJ#QF!rJtT* lS \ښH؊[U_\{zIV^ rE"~[-VYñXCU{[j/-nZ2>"Y6⿽B嘡&TCԨ#NJc3jBQM<+u:yoh((ZRn7K=Fc✞Sip,jX%Ph"h:!&pMlvZ}ڸ8̨5Z . יsd*LoKߚxۓkE BW2CuՕƗqM}{~QqzúOQn>t*}Ca ,Qn[ӽ_5ܐ;NwCx^&z`){8K␁;A*06  4Ԇۨڵw53x_Y3GJ8hmӆ3z}n--6-4F-X$l,yl,"acy~I@$e*0fSCaCcBr|$_h0DwYݦ֙J[o Sb˲pu򭇆BOS&oj -J]PB^e:^QzgsƷ*]Z~z5ԊՏk4ymVh"O\s(Ҋb!-6 b@q, $a * UPшYl,YK1ˊ?xW9>~#sK%l W vwunmI M.ŚnȰ;-=sxњUon1vq"NJfI$Т:0Djάф(gUI?Vub]1uLKH)]J ]JjR6˔]Vg^/qm TߢM}Ż.OF-Wɫ\ߥ5 :UʯºU.ZmSm.]sVkiJ(D$Q†p(,GL!9Vre]wֹDa@ d34hXs*zQ؀C%r/;37#ʥ{s2(Yʙܭ nFT2ONт jUs8L9u*>?4 J(٩7ɩ½MxsUM S žyqUF#XNMTU@o]x nF+@V&8WaMZWjPNZ`V"W3qVkHӻKyEBy JLXy8b[Nj+/('˧Ākuu Z1Z/ kYj/VVV/E'~ (LѱdN :-ֽdUdrRP΅+̘\|h NcVn wT[U{,-Q:ΕNK `7q*B'ͬz7WXKE%ZZjΩ@| N-E N$ >aX(K:Z.&uRֱAh`&HZ) Zp8 Sb@,tdGGeѾ֝c0=Ƽw" ʚV?_txUdc3؋͝'nceMŽޭ&~=ո=ӽ7t v|?n2B?Z9N--nٜ-D`r[kllXWUVFO|l)m=$;;gnJO/븾duTscxEXG?߁Q7JBRU(8*/{{B@]luX|tA"j9OlT,vDQ>ܷ%Tuu*Ǵ.M&ݴ{ɸV)dh+J M7rCc뾣iq Y:B\# @F"!o( L> ",n nv1n EH8&?;9sk,~%vvXXw/Oo{LE|q RNJC|eꃍkBm5ڒwo_Wnؓ7x*C5FoEWMm7TNޤO+nOd0Vc=s;S;?41j敂h/vLlvO;dCiWGpNdz "iw8saE+Ⱦndm9?Z\0o'S '$~J:%C:H[QZ G%y^|#>&~bW ͆cs# Ic/kєo^(61z;lGHCW+UnjlZ_%TjTJVpݪh i//}w bI=>,ÓIeRII%JFI^JJ*TI& \RyC1x.\NjB-0Ap]^5"[5R|xg\\-9x4rH[s`ik\UI8T= , PbՎ FC#j52l :G|OmK.Қ<~tM)\Y Qd SPvITlvMT(gRLl'y5F(.?6H%ZMd,sbGQ1_R6ʒϷ(חgr  &n+>t 9Z[) ԆóȪ)*^,[lݥkߋ5IG`tAZ$Th9®}`)9g7PEQgmI EHO\ȩZ/ܼ֥+__źWKK;\|1rݸ|Q]mc}}{yGgDlog@+Q;V:7{O7#%ŗyC5Z4SueueAZPxj8|1J¥* ϲ5mʝ:~*SD(ѓH'}ͺqo vHg[H\X9ěұ#`IӋg) riRkҍ?~Z^(8?e!g/+MKWmg'w%c\#dlE&5/_2/r}kŲauC[_VfJ26JP]ިՁkwRͽ+-g=عbnhqMYE7џ 4)Q\0HKK6"/TRBԱjbLA)Dc4HGA4Dn "S<" s#zb$7&{(F4.;'г.`*#+ vϾ]#i%=՜TwnƟSo2e+ߦVU^5>[kjJ"_؃X mת Q((g?gU_|Z |A>Wt$si H }2FrH(iI2@chJ?٫߮ _ka2tXք6E}h s2'Q'dO ӅEE (rh 2<V?yFC=YZ r'Õ _oT>Uid endstream endobj 173 0 obj <> endobj 162 0 obj [/ICCBased 200 0 R] endobj 164 0 obj [/ICCBased 201 0 R] endobj 163 0 obj [/CalRGB<>] endobj 165 0 obj <> endobj 202 0 obj <> endobj 167 0 obj <> endobj 203 0 obj <> endobj 172 0 obj <> endobj 170 0 obj <> endobj 206 0 obj <> endobj 171 0 obj <> endobj 168 0 obj <> endobj 166 0 obj <> endobj 211 0 obj <> endobj 169 0 obj <> endobj 200 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurvzzzz'nn///bbG`V__Kww?33(3 K  c  z   " g 6  [NPfD~8":!i5PM%= UI=1)& A!X"p #!$ "%!#&"$'$%)Q%6''*&+((+'i)v-((*.w)+/*--13+..2K-S/3.G05/2261$38N255946;5J8=69>7:?9@<BBH)@D$J7B4EKCG)ME.HODFJJPHlL1RIMTKOVMJQQXOS>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 204 0 obj <> endobj 205 0 obj <>stream x]1n@D{N Xk֖mE(DEnNb,=~u8VV%}[9S徤\8uSc=HEux毟9x +u\}NtuR^;{~b_e~=q6uT3&*εhQA>*hMT6*h1D톸swQA+'U)*hb2G0%V#*RAK\ac6uDؘځQ{":aO(pfɨd`1MEAcr^_CK0CK{=^!%h!%x.W;\2ݗ%O7- 7NowS%R3 endstream endobj 207 0 obj <> endobj 208 0 obj <>stream x]=n0{N0Q$4ͦ"QƌWk,/[vS>G{:nvּTmtYscYָ?LJӞB)d8@pz-t+!R BSJP^huPp>*^GuPau#N^t=Nž{w~QE*$:0j8'>'*N'x~!>W4^+]me5)[a_R endstream endobj 209 0 obj <> endobj 210 0 obj <>stream x]A EVm6vMa!Eo_.>Ƀ3(:X|J* 0Kʊ*#םo¿?h,.f(u)G:1i Xdèʪ(G8Qeš"6 b|YG46?Rv%qXw>h@_a endstream endobj 212 0 obj <>stream x|{|/3mfw&d7f -A@D-J#R9"ZE Z9x8"WkQǶlyv7D{Msg]g&`C]8 !T,[e`.ڝ7 :xz-*u6r]n @"8~ͺ_y%MƪԐ*[nڕQqlֵ~ڲfWqT7ٲ>7~*[ߔ7q|J=@c_;`1MxpmuRIJtfrLju?kG}~&2@Vu`+@/Fo[@?v 4/ЈZڨ+Z` @j+.xa1p= j@;ɍ~ j!GyUVzЏ ҝ ӯt9u؅fJp'2UH;jV'*+28-*CA1xv9vR`6qѠi5jRpsƩ~oIt}qǀSܟ93uL}Mx31P0-c_,@WʝԲnزqٴ,rXRn)d3:mش^[R hu! <[!!֖3PJ̱1*ԂCLj] S6uc1pH8S<1rxukA[xС%-? 1hrRQ2 -Y5kz'cftl3Ј[FWGF c/eoo-xEDZ65[qH%{1`]&|Mk:[6 #֓&k C[`wFFZ։-֬k4&/!dEAĺ\SnaHOs?&0qM;+\ jhw xh1a0{D4j<^z֘"ĉ't˚\2`Uli֑5C׊'lGWAgO?s=zw5XZ{n7_WR)X:r; =w@\KzbҞ^7S :{Ċ86̣3iʑ~?C2UƆd$G?X➡|"YuLe9[A@Mf)McJC^7f!06pIbc w][,v,Z#hA%wXkFr~jZ3{`ޗ xF+zs(X$ޒqchz@xW8@իPbAh،4UQ=BkVb0"Z5du镐lJzLuϐ{{kM;bUWڎ7mێ72JuScf]d;(d6ZP,O+)t񵋤(/* "H2vdƩN=~Cvk٤Z@@P}_Sb!x}[91IK,b /ғX!2qf;\a&~pBf<߄Wk2}`vMh:O oVP ?Z>-h-D YB_-ar8B@T2:a ~0M0I- d| =8q//SQ7 >d+|$B)24ܫ WTyWǦI-/7ۙED0 {]韫ih[V8HܥK)ܦ^Ot($ I.r,dtЍmR/>Z*#H)JDFWPbZ̉։_Q!$#G]Ҷ?kkc˷E W\ {33 o!*B:q3 Z:.5ܫjtg ZD#;=*I{(ᒱ>9f"1Kj@y*Mbe"^Ld}NMu$7sh1佮6ʠҖP]Upe 1iЙQ n@O&w)Tئ%&ͶAhѐH2)e:.(h(zPRa"p8.šO;4@ׇdNmH)t 46VVŀREĈd8#T񯟱X8ֹT{pWeMǫꪓ__uʻ]~%n9%o'mRZRJ XB|^3 rѢhpӯn"'rYtut0JGC%A"6(d0JIe%b2ylsp,$PaaH!}?Ø>ʥrȣ"]Y|o if$r.G'r>ӇFWԄU!  SG V? "3|uiRt:XPסdRߦ+J?8y2o?zCb~:$UT;׵OfztI.N&e Bk9eiY#dcyUmƽw_~)nWrY$`< ˄+E,D8kQ2PRJt3J#IDOe>iH-EGΎ UJJiV UL#~ٴ +~1).F/:<{Łsr{kwW\k?Fr;?P& yuhЁK(BQt,"5$u:tÔR5LQ^ ⡊:j>u RQЧAq0=jK:aNrA0'lX=Ś@>JyE|aMaĘ /0~S !S(kؤ]FbWTJ $)ZD8qQ4>"$ RXe|5]d/nŠ&ài~1Q7ϩd j~+C_50X8t2q7>YLaB0#҆BܨVFmJ?q]o3uUzjھIu-]\@zu$8V Gz8dbؐP.a6AMKT:Qf̷-լllm lp>-H,d.,Fueܭ{T1R8te#1w;"úk"cL2w̳xZs!xJdTd9I,$`pe & 35هb#=h+y}^MSs:sw _25~B*8lBϧ߅?_:dI9'^ҰUR֗XZ. &SRJb&cN&)D 0͢UW/J]]NL5tM~x,^m3EyomRƱ搗ЅQ>f eٰX 0m6I57Lq&cY $xq4$~6&J,Jc:m'+zb1cl1_LMʕT.FT<=I=Éɂ,0„U?kŒpY9/Y,}S{&:Si/7{/'zW(+wl&]?U'4:)F) Rޟ5cEM4Ll}OO{{;kڏ(-'oؙ$Y)%e7L{acCw6AҾW/Ӛ&**=o^ ;yp[Rbߞv*ٚV lɝ}Ϛ@h0Q*;EMIs ֺv(]~Şz8_6dI&r|(-^- r1(qbtOXy ajƋ|Fҋk4Ǝg54Of=^53'zWefo-<,=9eBMp?POL11DO'l᧤TDe}[yco>Zt.;٬('b\T`y!|rRqk6Hݺn 58-vj\VWv)zgϲ}}tm!C!C!q'`wMrfMyYaB.Ų2Lj; On-AUA}52x7s˹I39ĵݮ\~l4ތex{E6({vkL̝Kp4 H5G7%zmai w{SR+{[{n56_nU[΅[%nHF+Y\Tp.}+Z9ܽԢa@ũ֒擕ƧdՑ]Tp #"VVږcHJpCd@%l!~ d0'ޑxEp-A躟f kn]K/ ~ʇqb)d|ADXg@hBk$ sʗL=6Vx?}pQˋɒ{љ%<Ԙs#B0qV%S/Js$I őNg!lh+}.]v s?sYH,('CP%[MLp5)Qjm΀ѷF󐓙H~Yx݀:(Mo|3*{~dH_Hqמ$8'h]-' kT8 :Wrpd Rf #jbA&'ec2H9\fFfus̼`Ty++@%GyvR>}dsCA"}lJO9}0б`K@pak¶w3{cc~rQL3ӎ4cp ošxVZ_KRM @?MB2 .9Ch1xLcGmn% ]t^Eq! isn#9JmZ;[[Ha8^6 ʛʛG)<{Ϸh6WUof㭏>0Fvr eZnϾ&iS;چrkzo =^Gw|9̲vοͅa=o )rs+o|s˼Bw2(J0N_Io@:Ju-/'ד--3r!c :F/4hQ}W`*6y`"sI5$I)Odc#q7<Y|NI/˾+.36^Q]=n34QꏥӝDgh N}O,}ʉuy'hI7][t!gPs.Dj!'D2OTJ̣ dӧq16VeBb:Q\`vnY4[%@1:FX,INezek^+@nhX (Hsl⽮d/X@OGu:ãz"Lѽ[XVϖBoEkli&m2#r-̾ӎ>,Jl@?į~pI=2|>stream x|y`Tٷ7y% !D6IH" `@Z P*.UкTEd mj҂Fmhf~o޻,{}tӦ()3e̮Xbl\5lN6_jlS@|uWgۮW%\ei?D%HcYb㖯\kX `vE=ګVvɝܾnQn?/~[223x'3l WqX?VA~O&iZ7[0+W(|IK2`0H Ϫ7al\RnUf-{iųfQn:)_ɴH-}^~a̦`3\3ex,hp3Π~H 2w✚q98J_!Iz{I̋w 83O8up+c;\∖XA&d-b7 Pʯ.=8g{dveR_g^;ӀhU!KlE8 _=$s4|ꤦghrnǃSW x7SŁPhԦYЂrQx%Ӌ6|N\OJɥ L"MfSd/9H& S* 6op3fn5w˽$_Ou PpJo߮hxh׃Ń4ee>%[`1]i1|2#d5'gr5K2'Ez/aK 9J~ON/(QV0^ES=<Ÿ$WUpc6|<p'y2~S\_]qDÈUK~ v4qZ}z_i4@k+ڮ"jcנ"n6 z;m'/,5zIs{1Xj _XKƠބ("n;vEV2=`oOQral KC?@fC5L'? m9l#g<A3K5udy+[j(XF]ECϳ-,&B3Ѕp_'ZLnA=]H$9 {[@%n&gWW(W%rCB y1bZPkAA~Ena)1q:"N ~ŏ*c}& QjP*J0q1r RVdZ2|q܈ܹm%>q+t>CL E'2N'"A GjI$$3:25JS͸6EԼW2<ĞFB9TӍqJb4 QjV#>@7PMȏ+a;p nA' 4".}7DzNx(9&-^DJŖ3O{~r|lJ=D!)Տ=zdUeEyYiIqQ2Q/ȏE# |^tmVY0 zVV)"mԄt5u Eڢ$tkuXa- &1DPgn jN* oL R\qUiӛ{E(" )KhoR6TmK`wpm UkNq Z= ӮMy^/Zdͮ{E5\5o4#w4mx {T[nde&<.kYӕ׉=t s MW A,}0 yet _7%hFszsvoH%mc=zXOPϹ¨{ :vk%Soefi6vx4V\_64s^Q/'R;75)>_WUjJBRB%ټE+I̗,8Znщ 5/ 14sNW @]] ]m] ҙpPw@$ֵmHL&oj|%d4j+qarn1cNC;f6k6Ƀ)˨2!2%HBz܊!Jpnb6zݴ!0wgVb\#坬GSQ?; 0&c>G y=&F[Ø^(}HCJRRϥpq{"^w2(#nX\9?Wލe![snSOM4Vm_`D8\Y+WQ"%0=zg0avoGvoہȗK*9 Vj \+g=Cc۸Yx=rr>_Χzw\_)Wz^dXs˹˹n$5B˩fRMː² YMۗb{<` =cWa{>a1xxxd܍i'2e>1Gn< x^C 8N£1ñuK\8T+עxЧĤj1rUPI4 S&^'%q^'_N}`2+tzaO7X ݋^hô S'ۣp*cځi&5g{$T:Q㽵rY1"[Sn{G6a&)QZLU'az ILȌ|dF>>`>/Rġ/`*vF-@J [ԓ ? ݘB2d B8 SKFƎDOńt rs m ʌ{s#ƴ;G|< !%Qz[؂f<64l{t~ʪUwWSuJu.IZp8--fg!<0n9N%9wJy /3<8p< *X ۚ-jqT\Hf)l 0<*WTUjXU*PªʦQWkjRͫԶt攔`+M)Bɳe9i&A5ѦHSBhjN-84AqDSZ\R+s\#n@z XWآ uYrR#zȓ.8p5.ߎ\?vttVߎ5k111At|zy} 3lބi\GGeڱְIp%?L3MxGksjA/l2O/V *ЭT~/%YRG='VOɕ"V왿5Fe!Μ$|f̼fnfNc}e6kq6-"-fV˳Ev$'VL6gZ<2/MeoU0r)p6l;*/=4FkM?(3 &nِpdNk{x7 Ãwu!wق{/7ڛ|#JPkweטS'%כ4j( XBA wL? Z禮tIb,Ue5HVΠ9 :˨:sFY)R6Ye l.#G[I?iLnsvjq,V'_zm6.;hK V҆yu oHz[]HY?&WV^#')T^bx{ǟ(x"~PPchUB>.'lަcbe8y#ϓӠ1 Pݫh4߽Ci^X] ED H Yf18GIe#Z/`.$qlD0Y׆ Y~VՉ"c7.VEFUJReYR@5@os!=NogڧJ^0;Xȟcs|oȄ>_dj'niuN+c2]u<2 P˫ *˫̣#F&EzmUW?^Dk?:?Ù)1B(8s%J\m% \ N`A,FdlUgSFA8o]҇" ܣ*Jg}3+E )G 4Ncq5gAWYZm"ZUaH*&odlխz5qȖE C(XV"N%KiFBkn*/& ض[y=I3f{hǫtW>W*XUM[g0["]sZ[i8Y~8~\f y W ρlh^ ݒ&mh|ŔQQ1D?qBX#.>^84K[%Q'h >z-dC"Z9'Փn]][fQ+U>Qʼn GZp:^;TJWTJ*0ḁ7ɛ{&<9|$g o_ej|zr@D=?9?I!W i.EQ_qKfJ`JlŊ7Vۄ`@# G%M^k!IgS'q|\EQ^;y6 6B@kV}kL} [)4>. f^eDjyEN$ZWTCkn8!''Q{õSK0[v[o ,:A@O^*:RtXܖ-ݍA!|ZD0%}q ~|~!uߤQ+^o0W2\IgKZxvDmHyWx:ԅ&\2:""޷|jB *cnS$u%J{C Y2GN`s!>iIi -N*W% N)Cy|_Qii')4:( |CD@Ox.TWj$FwARM6sZ'cu4[O3)3ʮ ɠ[d1qJAg$_(5&HviїF HL($"GG| |Y-b'pmRvI"k .;LWr݆y!˧<_G4ԗ|.`z;U;v.Z0Z;={ݿKw, 6l#* roUڡ[ñ[UHJ `΄ީ]ӚM>DBԸ sOOzxw51ch,N]n*1ˈ*!@6 <U;_bg_]ւ(1 1CFiyD"yhsu%mߑx)h>zHۧK6a]Gi=_؟4\ܗ#5 "VX|'=!ADʼ:Ni{;X1Ai/*u:UL+ >马EeހΫ>P cR.3"׎YD2 0laHZIjX6_b4^ X)bpaKa5i6}݅fg4;][GA!Tʑ򲍡uX.ΝY6p[?p|tӵk~v8|f馊yeը]>m/mU+{|qъDB&|*OG"p3 3_^aVH'3-)_W)k )*I4X0 I3y<',SY-ѐ*Y"BI}JC(m4"$$1O:  QoIX/N;=!=λKE%_ܡ<<]$&|^Jm_KZV_+y8FmJg4"4U| Uܙ3u:+-ҔP=~bf=sĪ*0 W!ߓ}9᚟{O3A>k[%va*1}.MH&r> T]hBS0<AM2iM6_ypq0!0nedn OiiwM)}NeR+L,ંBG#t?8>ΟPV|6;2R5KWYVU*rbn-I{]N_о3_n#)-emnE 契rNl{,~=/ ~#_y1F_?v0P:Mh>Q3#.-%{O"|Yee;zz_{J[{Bt ۅ]y?c1y pj4agf1JϜbXgQ $mb ygbs' 3Z~&\q: yTvwʼDt>І26 Gs(UPQνs=O.}Y;KcVBNG_24']C%zf w_#?iB yG'I8^3:BV-AoFsD:s4NiB<ɨ)!M6& ;L^)`g|2+hJ9rFkT[ b(D'.hg{ñǽau}ȹXjh%c֐*wLx{>u8q^}|vÖ ;y& ap }?{Y`wN`wyf)Lk}#+WJfO0a9OtxѬ -(M>` i#+ml!6X0z @aS"$~9G' fJ[_퓎wcF%~vmێ{#fLm6 峉9gægfVQ)*):7&~8:&eAi*1(KS~?@i"I)A5.X"RbPYҎ$ z^U>Pbyjj=F3LG?bȀBpCΊ ^h2 o<@+[/=9|,sci qt`%ˏggacjK)a`F =qj˛53dTh[e'7g35E KVh9bs*vwVTР]Bք(e ,mebPa[LJ$yAjNMJÙ4S55{4iOUNC§_p*Edp_Aj}ݺuH ͜OLO<ǯ4RVX(eo BmTF>9rDCZUi( ِ]"U&ΪtCc[Ϟƣ[[/?$#ƁW_"]Z8G`Wm1Yh +uRуA;JVKwhR'T{T/U^P^%h՚: bPk4M>5q3[dA>; ʾ8~З9cTbd"*TʨWʗ5Iה[<3-V>9yFO6yWb-W޺vؒR ʪ1_lΞxԁ_nE>;gLvGmp(^* ݒrB*AhksgG?{ɽ}J^ S C<ڧڈ͝7}}* R8aWY Vt$.֤^O@a~mR/pi-\dyCJ5zw5N$d[CS>4qZBNy>>d)=3AtKfJ9yySy»}dy훳1^_;ps l?@\KQ>yàq溙_ \CʖE7/F+?r݆V^E j0Xr)M[#V-VTGlº$la {*?~Vy6zR=::rYdY.ۮ2RCu0nj~/H nY) yP8 CQ*KK+*CEUf|!chiCf]L\\.-Y1Fϋh(D#`U2lZA`a5^e&YQSTLR] B6BkVIhdVU QVUvV`eie[%W(o~\}Vi:5T5XaRq<U6됲jn_qr\t]|qBHfki,}#<[:VP=h,v%k9k b?wE(*P[;̩^Od2xR~<܃mSGO**3_gpu+spMo]j`L/+$<|K&&"$Âɢl+ dӗZ3~kXdꃟ{Ձu-ʆVaՎ·%63g6\!Y NpJQ;G @jǹBl-jM#MQѦZSI255~~'}kTk|r_7K(Uh2z˜ѣƄFMZ4S/-jE횜~;8uavםG_\(*969_0+)}Ʀ nh WGn?D`Gk?~E+aOJ&줿II9%KJVqv}'/)jY1y^MbS4yT2k"U !ԋn>Ml$F 1% ozniao3u NˁyCB @vE[,x:GWl-oX!moRĪ.vG^굫g4f\Wo{vlq޸"h`'w% `",X6 ꃆ1h T[F[KmK{pV Z)Pb S`i͖A-;i8y[\Hwe).3%$h1,AfE@sBjtI:+)1כ9sYV ZVZv['{ BD:NY=/طd8vi۹yq7kR1O.EkWȋn 7(),jY<630rJ̟%tD_ttw.dt%4uBd ,,2UWNUv*w*qY~k~KLeeQW$﨓Dmm /@~6 z0eT%Ra.nʢ͘ VQ_\@$,|q5 Gބq EHw0 ħ Xsa™w7~d'htp^Խ[j3wټq endstream endobj 214 0 obj <>stream x8ktיߝɒ-[#K~H<#ȖlO׵*P`&6v:!2lfIndDL*:= g6I[ppYd{HvNd~{ȇ1sJ3 ؼc01 |"C3$ I'~\w ##BP908`v{ #`fJYBP>op~K=пޫɸjњ[d;02N_L-kKNHu 4&?}#_Tdi%1x ,qzC WPl%Zx!^ZBF>\* z "Ђꟳ[LC+3KB5! ~+YG6GyF߁m9Sf߇Y! 2XCyXЎCFgJmE} g. nkxޠF)^2yWx.gmMuR\j1V_jr*T9.nyD;~Lzפh]t2^!yDnH`Iyb\;e'ݒ9㌐R7kv*/7(s=nH!H T$:E& ]}$NBJg}T8m 3f0"R/[L)LP޸K'%ahc hGx[cX^bpqXÇ #<G(p#[>fJ8%KjE/Ydt7/ctjCR cX) .ӂz=O{Ivɉ~C jxsZdO%Z2%[{KTqCeYY0$3n ʆ 5eNH$.DvFH<|$O&ҋc T{{r(]8+xq*E$6>@Vc l6[ٵDVFm Q;7b2ލxf w869F̢v\iz6D34s z],Y+KƖ$2 CHT-l1ov|xw Ϲ#uLzm}I>$0ʈd̋ݝI$Ƭh,׉0:'e\`ѿ(1z57Pٺl)_]yڼՉHOfMQIJK&3uLZGN] w3,# ʊ4RDcݪd.Xӵt6@;whR9mjbPx~L|x:GA^ײDmzU]~Mx-߽ZǸccO;yLOOk=NO N046U}Z)ħP&G'<9pIi>*W@-/#%hx/沁#7 f (1'_"Khby|y [#7)#)ƋրSzCY|@/Ko ݠNJŷ-7˸7_6rlfaOsT Ϧ42^5pw|>4QMp'k$5cNEܣyԁq! OQ3v0VlzܗGF)qvtr{NWAMҡ x+K͖B)?G̥N CrYN_9ͨt@(W-85՝jZzNMy:a05NZMVi.? ۡE27H~W{Z%]vm!+QDtK4CapheF b1MC bOFb¶~l?R OHBhY.#RZٰ-$;wY#KH ayTY۵if(9jm5ŀܽWf(njSmyDt㗆_(~k [k>Ahim<-othIrhٕFo#U:ZJ8<~871&oς yǀ:" +{Jac*.)'%EFdC Uȯnu-}kf m[[U-ж\]Ͱ.iV<\xHqVYY_V :cAA5f^o3'lW6jkw֎Ԫ,SNze!yWYeuwni9b?\,,GaQ@r P! {ɮڈT,( fP|<6f$n^;4aaۉ;? k<1_YUv^jk63W~>\{YP|#8ߣfp@PNl_LM 1ϓ3bn%hFS]%Dž<%weQmR:G[8hJ_^f<9ҵaddÆ6J867zVu|{B DWl_)A6z`Jbn~f%csltO:pRoK8iGGoYØK6 {?+0ٞ;mh*[f7"Hf]ޝu#qdi7%eѥs>s:pG%[;Uf} p,Yu:RX8[YYu~}(S.œUrΪ/5045^2+i:[@r,8LUB2Y*Y7 J)A)216q_>Mx o=Pzmջ.g,V4!2sOgk2]}k<75#TߞμGp"ڦThMÄw2dKhJiɐsӱ.C8RRB}L~K{`e>E 屺opsc F@Qh; k`&hMJzv_~S}|({X*j/P endstream endobj 187 0 obj <> endobj 177 0 obj [/ICCBased 215 0 R] endobj 175 0 obj [/ICCBased 216 0 R] endobj 176 0 obj [/CalRGB<>] endobj 178 0 obj <> endobj 217 0 obj <> endobj 184 0 obj <> endobj 218 0 obj <> endobj 182 0 obj <> endobj 183 0 obj <> endobj 221 0 obj <> endobj 180 0 obj <> endobj 181 0 obj <> endobj 179 0 obj <> endobj 226 0 obj <> endobj 186 0 obj <> endobj 185 0 obj <> endobj 215 0 obj <>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurvzzzz'nn///bbG`V__Kww?33(3 K  c  z   " g 6  [NPfD~8":!i5PM%= UI=1)& A!X"p #!$ "%!#&"$'$%)Q%6''*&+((+'i)v-((*.w)+/*--13+..2K-S/3.G05/2261$38N255946;5J8=69>7:?9@<BBH)@D$J7B4EKCG)ME.HODFJJPHlL1RIMTKOVMJQQXOS>stream P scnrGRAYXYZ acspAPPL,descTcprt\wtptpbkptA2B0mluc enUS8Ghostscript Internal Profilemluc enUS>Copyright Artifex Software 2009XYZ ,XYZ mAB  Dtcurvcurvcurv,curvcurvcurv$&)<@EZ_h~,<MkxFDc N8c#Pec[MrcGh|Q l  k B   / . 7 o o ?   k\t9iDM4^JX?*$ |Q*y`J8) ! "n!$ `"%!U#&2"N$'O#J% (o$H&,)%J';*&O(N+'X)d-(c*}.F)q+/|*,0+-1,/33-0,4w.1Y50 27 1-38X2T493~6,;47iCBG@CpIrAeDJBF-LnDGMEZHOwFJ`QH KRIiM;TJNUL,P$WHMQXNSZPgT\ QV]SGW_kTY%aV2ZbW\=duY)]f*Z_bg\,`i]bk[_:d/m`enbSgqpcirtewjtCg lkvhnwjCoykq{ms7}{o't]pvBrxxo*t%z2u{w}ygcurv endstream endobj 219 0 obj <> endobj 220 0 obj <>stream x]An0E7`flhT1ȋ!>3.3Sc?y5!m̗5s)Wc? p4]7Ͷ㗙oy\tj4;jmmCZgI%`m uQ:^U{ 39mUډ}@tPL岪d:L]PLAs%j*F@ dءWڕ{)V4?//YP=~e endstream endobj 222 0 obj <> endobj 223 0 obj <>stream x]O10 1"0h?eB:g]_:6VE$hv,t2qAvك?Brl zCkHQLUkdOg]uJժ/%Gs$#q*EK\1~ >_R endstream endobj 224 0 obj <> endobj 225 0 obj <>stream x]An0E`'R4tEc*DȢOE/[~h<kY/Skct_R./{)!tsQ^3[OISosLyw.cѦV8El8E ZFmcqm*Am-A Zo1*AxQ1)A˫:%hbVWJ#E0> endobj 228 0 obj <>stream x]= wN KVU 0C"C kJ#+5˼t:T['Ixw9{ dG/F -5Fp 9*L8/k1(KB"@EVySଳkqBⒸD o>hE _f endstream endobj 229 0 obj <>stream x{y`Tٷ73odfLJdB^HHd_DY "Zp!*@ԪhuiI2PD\҂~Tlbf~羙}{sνᄈsN,U&)sZGep7Su8 YRU<¹ W\q|xȼY׽7qA`m8p6Md[tOSY.jY}ѬoAY{՗,^vSz ~c[w_ ~t4qa/I锺&'ѩ_`[9y ,~8dDa3* `LVAO GU{i_Rm'?"e<.83$X蓮4&n0Y+9NΆ_(=?oB/ 3.%dYr n/(i5@\4\* GHl0>x !BH;yWAߡġUc`co&&ut:7}_I?= n­pl>&It2|Pa6Rs38 T!D&wA!9SW#?—~Ԓwir7%UW@rGT\ k8(ߑJ"HdH*voHu"BEs(?oW‘J|H&'c%T;Sh%Ln|ΫpgI:4>"?8i9ja?Xx8ѝHF@{Jdhƹ|Et7RrBV2,&KRr+ EcS5u "tLwC=7-n6s?_ȗV~R*NԾE}]8ѐIb]ĉħIC@3PC)o(?/+_/8=W>4G vE^ 09B%'7F-Lwxn1rBnW Zmpk}~>)*uNj ՗jA}MF\ xqoҗB(}IߥQr;=O~C3Ln7jd?J"5[ԒZ"V6}qS7N玲4yӑ(i7sG6:o⇒`>|uPG"ރȡ"n'Q~T- E;XC({2qԉԪB*?#S?ͭض6BRx#G1 DdqGdLt;PN:70q/ˣ^ɍNr?<2RڈɨRS*Z߂jmt!'sMu0ڸet8y(_Ǖ#zКԫkAUe/`(J\"Nr-'iLNt'2"F";I԰n%Y Y4(ת{_/qLGy7lGM}+x5RsڞGķ *]}` V -tՄ] }Pk`=ڀix>Kr]K_7|Ľd WDȂĎO\ iC_Zr*y"۾c8ӸMa`,\7IZ2jPeEҒHA~^nNvV8$ v9f,fѠi5jQXNk YY0뒆X_ *h1eļ L9)_$B0 Ѣ`c8;Ӱ!*JyR6aYp@=!#u46t}}M_Tz XK:k(Q 8քy 1O e7κ.6n$5H,1|6.Yxc}\٭uf͘f5gX#܆k*Nn^dՎ5ض.ln9p,141Ow5OA&Rnd- 1]xXx^ǂVd#VH]^/yIRnՐ)BDŽ9蹼Si Fӥ }JIAg (K؊W@Ăsia|*ڪcNLpT:C*[;pחJ\5/"XAM=8TWK fH></I2J}T=}] DcqLf==Qw 1m΅Ep nK7M 7>-њmӤj }R^?t8rdVfQ+B}]\ETZHpxLhzIœg(%8,ؐ-4izGh::;Z;fœA!ܱ%'{b7K#CPZ) ;evi }NօM}"~4I˟W8tE q'(u4]8fCۜ5wkJNL00t X}J6n?06&҃-=҃-=RŁp{=]Y|nOրo\7$1Q~nXt>3߇yn]WuR:o&1Q|ǺFO) *--[%PU=z Wg݂[} oQڷQS uY,fn STnJ׀Vn2NS۸ISLU*wR^+Z\.3Xr ( 7vύRq\#dc>,]䣹J~51oB<棸J*7`>, j-Lc \S܇iSJL0)k:G8=2pw-CS5e.c(*3G=SbpQAJ1ɘajŤy q\! D /D̃<@סt]? .wA+%.R'"-4LL0mŴjS=Zn,(%/32S;Rw#dʇ8\r>.9_DQtrcNabEb"1rsq|Vġ㨔L%Z%ky8&qB`0݇@/sHՖ UJ.Eu8җ BńtRsmʔ{jaډI;\-9{{ހz!7ĝ:rq*VT2'eWqĴ1b1 *#)?bb:L)pC $SL{\%V>< )ŵ,U&U=Rin0Ւʲy&@C"rfF k4y\MXЈZVКF^ժjA+ƓAD25  U6iFA5ѦHSh}?1'z+Ua$fkIܱA&9!Vi]3{k&M$k}@H]|鼹ɓ ys6:xxÿix#Vp%&N= &b*:aIJiqk׷74_ă 7e A /e)< Fjl蔤~rUpqt 4H 9?pNĹm""`9Y6Lu7sgHG9c89nkՇVJ=* +'M\)5tG5g54wU΂YbYl;E==k{YԣXjaX3JM zV<),Hs-7b8gĺXj2H/徣Zߦlq~~-KϲenvٵX|ݴc,e7AQhm^ibe˚o˖&.N~g&.Xv$#-[N!.O28 Egae윆B740S.J@ųz {8zuֶG;Vwdp.:/:F>:ZCG"e*Y%4e -O$qCAxNYd ZЈR a" }˙3P[VJZDPNe rXrVyuۣ,lbT_jB,"dT??:lZ_^gπluh^i)kQaTMil>~V:w/tZj\/%"o$7* Pjs 6MRV - RJG$EG%*{э$z} $_DţzpmnK=VICJlGJ|!N54mQ"@\[okB*>RR?Ʌsλ;鼧wL6XU 1ן6&)/m|' zAc &4Foc$fkMv."pa tRSJ̜"}$MFH3#es;9-=l+ Ė ]W,%K#̓$FnC4VV5jMlZ>zq]:)7TTU=##Q?ӕ,q2Gf`!_k1D\ . pxB;Ɇr3m5p81zG 1)Ad{ *ҞbEL3cϑd 0nj/If.Ǐ5e43**"ʮĮr$2G&"o UP8M`E /9)¥~I1ߴ?[\ nٱY{Z^g͏6uS`5XSW]1fg l^/i6NBo_KJ|V&hP2Tu|Lx*,%~S^)Pʜyjښ_v-7s܂nΠ椣 7%?~Am8{w4 n'-bvͧ E+Rl,Y!/sz߳JZ[7i'ӵl$p77CJȊ'zf=ptǑx׊s!&5wg}dn *&XR[pBZdIiwh ύZtz-h3ܦdن=)-Ņk"*l*CyjwQ |q'U5:u Ez@#}zoL̞ҵf{G_ :2^a\fv*23]x`ZlNm4LT]$hsregi1*d 6ʐUĂ U<9fז*-(*d%-;8Dt.Y8橶~7^X_[&UV-w'/ٺm(/ԓu;[Bͥ_Hyl;8d;ZN6n5Vs%d:FaN tAoz^0zP .hU ML֘#ffطI"D%HAV=l %:6_ֻϿᵺ&vi!${Q ?hTK;],]ߘ&Ha`PMZDJZi4S*H$rW*YbV)5 *?#GRD^T,F)[Gmy&6ŲOv4oU$B%&O\M[MyTвQ *"~#=y,m!-Reکb%f9F=\҇1* n޷>3\zmZ6W~޹cnq9+}d=amnUo<>︲*p3Y$^:y$dg2% Ő)vOq:B~ G c!NΑtA`I.27 )Oނ$0laX<%r'{/ec F=7n>33T1 y)5fțw]3}{rh'gT_jN PXU)dEb\ZhXNo_x1;Iɳ]!/Ly-di3p4)G;H␥F  Nԉ>UxK2M>b߽r(qeX܊E0ȝ&8[,x{ ;Nf!tU,G!8:{t`KxNR/)d_Wz7_ȷe{;-CO9Bpj>pYdz`UyuƯ]R`6o Rcb lr(H~sPN6sF bQ_허2OS&f'7b;N>ԗ|C~Ѓ__loXqZ(%N$B;Ҷ%Ax[w>0:Ρ|ds_t<?wu`oz`oaz#,˝=CjVo0n5Srixj0NVo5(N>]`5&"^H]"{{zSX/32]NQ&aO )L=L0/H&hE?3׾Ug7 n&NoT7ǟx}hc h9zm`qt 8"gqv՝eی]]W"#z֢3!!]E18V'%(Yy!nɗ2lAOq[ѩ[/B! z)l*qȎ%S|y3u2\KPChmB"eIPtt1gэөtb=1- §g|SQ02aUEt4FnQf.zk0ZR+%R|sM*'k!Bm)嬱Yޥ)υT^E/D XɬtpS{_{ɌaoGݒ\wȆ G*cߤa_&W_UP>Bv&hzͣ8zua2"ԅ*r\|AS/Wo$mI#Ti.nk5RE TE;d/Q%|A&b3_ 3kVWUV( U`~!z.9&OdL& E4f*~# ](xKyb(j\d\esN}ꏚ h[imkZ׻H: 84F u`Sf&Ge kd # DSˈcj:N>vY$fo_RFoQ54\PC':sl9d63)ԍ/c{H]R@ e=y eY'1d3ycBYT~jҘDʮZ_+HQ UgdY x@ fY)z6{ Mmeo?ol4crUC+!:|cqYߪݚfnhv2R,%N@O)}ƦlWxm9@nbJԷla}m^Gc{mOT0mq+~AǒjKV^+fQ  >7WUoЏ}e?Sa?U0fa.`~a}z+.Qoshm1kP14 OuMdsIq<$g $L,/s~I|$3t͋I^ésڴ'r }貯[Y'4E+j0[0#۞@op " ê{h͐hѪuVfB#̂@[BŢsWXx!_CCA!g4N$z*iY[-"bXCTAD%n1` e^fD.Qi&x*-mdS'O[nNVNZn%ҪfØwl5__uP6+'x}7?E2QR7NO4jp2a÷oB߁(֮lSR]кWY}WU76?h~AҲGy6DQZMTyqlWgei Qĵ|<#kHAXq CuUW~8AN72 hy^`xүVʠ!r = eZtkD6t[X n6Sw]4>i k!1:VXr!Lfevsv.Ơ)hZ&U!\|q~x'"=kv$JLid!Ֆ2H`FڈA).Z=C9H (9XS{Lںv+%$h6La]vn#g Osz:\-`%ZN+g! NtIRb66" Y$H\1:̟kI8>ђ5Ț)v \-%+WvzY)ۮ[0 _z+[ho(bW3TdJCG9c%krυ yZOUzoԯk^>kCao6 qʯnR.љa^Lj/Uk fCvѡ鍼uqN0 }N&J K"dW=L3X@+pt#by^p!eVFڀ*?T(_m7ieW@6饽z&/kHk?Awp[vn(~MhDtv̴DR z"{?W 2FK׿jr$ endstream endobj 230 0 obj <>stream xUOA~%(-Deׂ1j#FbL[Zcv= Ch5z0i^`hg.SH1?x7 i89ͼ߼3KRDJ_N|oa3"bgǧe[3y_%NN? eN_zJߟƻ`zr#w5q7Mp?}r'7g03w/=c3 b+Q+ Cj<(|P(:R@Zc^b$7JĢJZl pLH*;xU 늩2k?0L=XѦ`i孡8d:n m#Q,HZYŤl%JZLkbRVuYޭJ|2Cc 8@ GTk4e+ZbX){S%HۡDX9V*ГDG2Z =IJ I*%mDy {wI}HTڒPsK\[n 5nRug*nGmwIz>PQs'WkKlQmҮK~Br(ϭ o}8R5!G|t8öE\#st"M-HD.TC Hq邃{81M*<0?sG 48.> ,S=tpC$Y'I'R[Bjn|gƤf0~F!sE`^wۮ$:l[tE-7-|3j#kLCC05d<0 ToW+R\*Y;_;4B-6BO??f|}N ̯q endstream endobj 231 0 obj <>stream x{{|՝93zf$a=gÖ,-ٖd;mI8/+MpB I\R %h$pwBCpx%eYBݖ ,ˇ[{Αϕ5gkΜw` Рw䳠 %+_;_oO|:|U@~54UʺToWJ#޸p1ڲu}ABkwD  u(nvc܄ }A@(Ȫq)p[$}ok#W}r*u''` W2x< GGx i A7kX{64gAA3hA5ԚG5+zO@ݵ}|0 n z?~ ~5\Z`E3=~G F^7E DeFAB50Rd&+pyY( >/y.^jZ%&#:FR*2fK[NەD eY:~:9tKeʞ,0g;6Z5ET!HrԡM]V̕s0h*}MYi8 +&Mʡna $a4Ӏ,rlID5|?!`q ThCWzTnrfMܞ5FG-D{&'&o-i&)EhЀh-QJhE 0s0gf~F2CdkRkb}оaxCK~֬2B"#-Cmo^=xbp3B h/6xh,ςuMdcLOx3]w< @ |fP8!t MLt|ؕ {z&Fۇ]UOtf;eGH;LW,),-y$./b Dr(?c !& b2ژOk!bt[:/J@#HCebY[Ɗ-s .g*8eXkIH}ZJ|{uvR|r8 Ko"(_@J dHV>0l12g}b“b #S Rw,r ∖ܿ/[=pE~t'(Hd2IgNCxgfΎO69yeϷojCBU"E #w!O&~dx,9xR9ݘԣq2xg"F\a3 0:A@0EjO6 9EdLyצœ@s^F(3 231@Q(O/U@ P1 ȈH+p4 ^'QWȴ{Ez,%re. ͹o@%r) uȧxr-"YCmmJցK%,F]r77o!^_IʺW.z/E{e /=8ཱྀ qwه~C|;yDw4=bEǑxn{w::pC=BNzVmzfgh OS=΍;ѹqgZW'=O,drg)$Ir͓ZCݞ=w\ۧ>>E:qsO8^{t} Ƕ>dJJ_z|2OM=':3Ϲv;"wEn7= |p>㾶7ũ{{R{OP/nw1{!g۵wu`7=tE G eL0d`ཌ38]n}ݡXmzSYT9{iZ\)reu:Ahu JG:)="U #2iu}:"H^2ʜ *WqtJŁì,oɖ@t^֒Gz&U?d}kj~d&˳:ZWvsM7oI- W,>jRinonėq{$vZT~y<ԼFrmnpFFP n,͋q;F @'S J!K$J ٘ ʐ؞VlmOw(ےS$T:SHÓpsN|Q:$Gd/rHQ6UP}{N"@Z/?P+SٷeɖbvZo2^eQ9Ycj{Bj'hh:@Ӂ'<2ZFբTDvt#yxŬP *amM] cV|lL2ZJ/i[ӭҏo;p;?xC#_~uv/\ٳ䕛o%?g%H6{ѳ9-VsLeQYZ]BCe>`5ا{TPLI Հ2KT&*St]KRȠєBO3DA% Nqɼ6AxF6K b4[mhgzr?&2Z;ܝ^F-l>>kMh C+vz*p'W U0GΐPVi 1AOCy 11X:Y䈃XMT ن8BJc3!,>F[[| %Q#Z_*PKhg }Sn%kL&TyϪ#U4-:`Xǣ@8^_wme4;+͡eRB2XCZ-vd7"K46w,YZ\]%Kˆ.mmy{Q:XW>֫1j1 @O:Nn’k7TJh1L[?0*'9pJ(L]I3 I$Ry#Nl)$nMH c|MRUh-I#y Qj@ rL?)-bYX21mQ(NCjq5{WMܲcKMΫT5}em{mFս AJڣcLen ES@âXQﮉ6Fť,TS!>Fm bYݡTJ:  V.z4dJZwj^M:P9weޕ,y=3 Vbc3V`$`|i!ߘJ^V=WbWW`¸15BYS3cs{_m:/<@ꮙ1~rKj _^gp)RW!R⼄S;#\:{>Ys 7$rnF #w`0}Mr £a:1(<_ҳ hTQ~?c KF26D0XT,L`].ΟJA,VZwb5P^WZab%B8F"bw) <=P O^QTXY<"1C%^;UyXMcڪP펞tC(yp`u/35Ÿr=r]|Q:ٔZ&;5v8Ot'SR̒4,-%dh6-HJj#=bjc^Z 0lr3e0l CL&Ñ`SȂv,K1p$x@ @X C1iaAiH]]_D~xR$ EB6RbyE:!~NIwz&-uGFO +3xrj#ݿ{K#gEoիَ^i|E~/c y9 =8~PFN6S+Ţk =0T=%Qk(EKK\TH-6SPJzσBLO3 5D-Zȴ ?'"_}Gd57Oe.꛹V]YU@òˌk8zj#-EQVB 4⧗ g-eFgM)`E;`>d*9Bq^hi2/g|+}}8^s5!{zX#ê[L*b,6QkMˊ%We=A*#gdY81{"tm:rܱ|rcVO&qpǃZĤT(}z1X\^LO~.qzQO ,DCuY",n@ .PGgDbV#<rʻD?wf_t! xP]2W4߻i]u;6dz(q[6 _-٘TKK>J3?(.4XqlB +ʙr<][\KiB4KfhbY3e집X N$w^¸bG (XK^sa3@'M Ol٪DCs)e,ZUAM6NY6t\ GVd$<+U6^bXQ$iyId%ܼ8Ģb{2bhzNPݫǵ儶4r=qz"B~>!4ќ\ZdbNy6gj<#*Mc0.}aWWu xq:JvXb;2b7ش(#s.Tp`l".E{))FL,=MwAƶI9n x֘ XU͉fOڝ9ݥ}U\{M5iC3BVEMꨟס/ C k0z XR@QGlҊ=TvSGBM&h4HDIGy6!9܉DU;Ժv~~ڿ`jBrwX{aa+^8LHMcHquuu:k;*7bn GL,O-5d,3.^oQ"߀!;܀7v0k r- җW%a U6[>[G9EC_y8 {܋++[RlËWNԺNm۲v8U]bWN%LCgL!Ed$vU)U25*ꭇΈ.LcYD/Y+݄|O\}d-vrSj&Q~2|1R$*%E@ԩ7AJ:d(I⼒DS9IZI IwbI&OYU|ZR3(Q)N*}S:.iVH4HR؏WwPoK^䌏O6}}:<y=c47AϐG<*H1_1k/!Oui0.\Bc0rX5T=-qFC92ΫU %G^/%l!̎%N_2RR\u=^/*h\b"B2eJqQ,sNeo%hj]gJ;)ń+E4.™, J{'lZ04'ӻAWF]EW13rig^@Bne^& 3/* >$τI`&f)j=c9m~(8|H I,F_dD80CfbdA@$\0.~QEb ~V &T/=K_^pr l@*yp %HP;HTWrv]|*RB`eoKtZÃH0@ :fߑZd/Sh)?Y3%>j+p>޲2ʣ<{udqNo~f?{&T^(ν{‹w_7- ^PPÏ,;8EQGߒζ5 ~~E- u=,Iݟc͵ɺ+A~yz#wAx9(ֈCVudb՗G*ؖjvT0.Be:m*x?XL1!3 5^n&a/Z ]"CC𶻸|ŝ }=FeHt́SPxUş\/ܖU|J%R+27U.wi=sD=& ]q ٰ\8T۹Mۿ־̙J6VJ|2'iu[ƾ%F9笷عh1 VKpoGjwHN9Ϙ22O)0^tc ^g5Q嘒R0) o%Y`;PEq# 3h!1݈x,.[Ifqn/z_ٳЙJuasuuls/ i8NruP7%34s>-mk"3fiQ4"e3Tu~z,NZ9rfjV?/t=.GWJ=5ڛ2\Ur z"bZZbUniVX։#6-vr7۽;}^(] k5<^>_Y/V#.ʵOmN6ٔy癶t#b1*CܟCXk enY)@K=@ޓV Yj]d-qtܘWyZ/'9HH,^!b:n)Ŋ6FU671MmGwkښH[d+k M펞> ǚ[ԩTC `m|$$4hld'){O7[Ee &d1f ;OZ-eĚ_=b:TĄԿoj a` #]5g BL dW|<8s7>N.M5U隁*%J-U&:X' dvnY]ӿzwgS\cQ5ͷkR ,4,)cJSϏFKEEMEvq[\ctG8*s4Zqz>FgfE*LGi&0*KUR<e.Jf>stream x8ktיߝɒ-[#K~H<#ȖlO׵*P`&6v:!2lfIndDL*:= g6I[ppYd{HvNd~{ȇ1sJ3 ؼc01 |"C3$ I'~\w ##BP908`v{ #`fJYBP>op~K=пޫɸjњ[d;02N_L-kKNHu 4&?}#_Tdi%1x ,qzC WPl%Zx!^ZBF>\* z "Ђꟳ[LC+3KB5! ~+YG6GyF߁m9Sf߇Y! 2XCyXЎCFgJmE} g. nkxޠF)^2yWx.gmMuR\j1V_jr*T9.nyD;~Lzפh]t2^!yDnH`Iyb\;e'ݒ9㌐R7kv*/7(s=nH!H T$:E& ]}$NBJg}T8m 3f0"R/[L)LP޸K'%ahc hGx[cX^bpqXÇ #<G(p#[>fJ8%KjE/Ydt7/ctjCR cX) .ӂz=O{Ivɉ~C jxsZdO%Z2%[{KTqCeYY0$3n ʆ 5eNH$.DvFH<|$O&ҋc T{{r(]8+xq*E$6>@Vc l6[ٵDVFm Q;7b2ލxf w869F̢v\iz6D34s z],Y+KƖ$2 CHT-l1ov|xw Ϲ#uLzm}I>$0ʈd̋ݝI$Ƭh,׉0:'e\`ѿ(1z57Pٺl)_]yڼՉHOfMQIJK&3uLZGN] w3,# ʊ4RDcݪd.Xӵt6@;whR9mjbPx~L|x:GA^ײDmzU]~Mx-߽ZǸccO;yLOOk=NO N046U}Z)ħP&G'<9pIi>*W@-/#%hx/沁#7 f (1'_"Khby|y [#7)#)ƋրSzCY|@/Ko ݠNJŷ-7˸7_6rlfaOsT Ϧ42^5pw|>4QMp'k$5cNEܣyԁq! OQ3v0VlzܗGF)qvtr{NWAMҡ x+K͖B)?G̥N CrYN_9ͨt@(W-85՝jZzNMy:a05NZMVi.? ۡE27H~W{Z%]vm!+QDtK4CapheF b1MC bOFb¶~l?R OHBhY.#RZٰ-$;wY#KH ayTY۵if(9jm5ŀܽWf(njSmyDt㗆_(~k [k>Ahim<-othIrhٕFo#U:ZJ8<~871&oς yǀ:" +{Jac*.)'%EFdC Uȯnu-}kf m[[U-ж\]Ͱ.iV<\xHqVYY_V :cAA5f^o3'lW6jkw֎Ԫ,SNze!yWYeuwni9b?\,,GaQ@r P! {ɮڈT,( fP|<6f$n^;4aaۉ;? k<1_YUv^jk63W~>\{YP|#8ߣfp@PNl_LM 1ϓ3bn%hFS]%Dž<%weQmR:G[8hJ_^f<9ҵaddÆ6J867zVu|{B DWl_)A6z`Jbn~f%csltO:pRoK8iGGoYØK6 {?+0ٞ;mh*[f7"Hf]ޝu#qdi7%eѥs>s:pG%[;Uf} p,Yu:RX8[YYu~}(S.œUrΪ/5045^2+i:[@r,8LUB2Y*Y7 J)A)216q_>Mx o=Pzmջ.g,V4!2sOgk2]}k<75#TߞμGp"ڦThMÄw2dKhJiɐsӱ.C8RRB}L~K{`e>E 屺opsc F@Qh; k`&hMJzv_~S}|({X*j/P endstream endobj 233 0 obj <>/ProcSet[/PDF/Text/ImageB/ImageC/ImageI]/Font<>/XObject<>>>/PTEX.InfoDict 266 0 R/BBox[0 0 476 176]/Length 3492>>stream xKs)p 3W+9ħ8bU MYt("e;?k;IXmnߍSďAi~o?/oon^=ޱhֽ^6zZnج`Q^`҂cAw<?sҐ"E f)_AD ؊/Unڄ"VˊAxbZa77_=V݋Y/l*bdܦj1NTMa2xw_~}/w7^?Ia6~nq>v0Vx_?itn/+aW8R$LkbIM<%}d4ij?B8u\0ϣ~I:qѸB%^dنTdhY#~)8ٌ9v?LG+-w1i9NV/Ɗc9j.jfZVڏUu;V;O*͜+Qd& 1WQ*ˌ;ߓ7^J8fE5D2gv]79+%T(WQՌ.ݭb F4a(Yuh@ğ)㢜5s]oiɨ.dG^:G-Н~^_K^WoOEW_[[k |}q74K nsc:l-.yE~$y_Mat,J$rn}oWUҡFxn>Z5jT#e~R VԄٝgh1n5.YU=z+F9YNlCc:;I>Zfn8y jBd2'qtRbҡZ٬`L9FPGqB-)OCIw׌s*\8@j$I3}OnVׄϙ6(dFhh'I;"Lgo3 TYmE=UJDp7*^6Y4΄{RqI(= l)=W_}ⅺ9.} 0rUD_рNtChۓ|g+p팸u`=xvL &~{Auɍw.+~t9*ҋYq61b^RGiAYŠg`J۰ ޽"](ߧd@胓<|zxqTq@UM&&u6BV"[eAIorȢ6,tc5Ts >z&ӨI!'f:_=Lq{4x;jz5Qڸ/BF]=B]ݦ?p*e%(7Ӧƥ~&جN{-LF:{+B&${l5&=΋F7dqB^kMDo@sJl:R]Η#Iz.0Hh>L[?e@-i&&iX<21j_Jxjč6u᐀VD`l_\_d@pa5gpd%#E cjTƙ;LĄІF(-(#2bþҳ 7&cRF#ckԹg+A*Z://% 8ju]y!JcJղA]Hk2q47Q n'IN4Xkn*(19 Q2x3!4~g\qDPHnH+'ruyܰ|-qfg{:'bh5ݷ!zMRL€ W⹌NW xXB=Ƌisj#6ڦbV"_aR_.cQh+1a-rUŪu⧳Dҡ[4G,3e0iIRzɰP.*53TxaFwZ~'@Oi*aMuBv u {ԠAq2$#3}ZC7&DVdCKM+adLj<兲D#EzD 'q{€CBroweGI 5#x$M:\fZxL>ļYdcq]4 KqZ+ K3{~(0*_ "[_~tSk%dZy BMt1Dh6oe TJeKEQkI} f^eIg D0>[)- q"%enI[.5c0@-sdmJ- }K-lY,$H8#Vu+*jMmZdQ4X 6}6d 8c.R جsraQR&P6xWX´<9Gr4lgE'Hd'P+)-uI}.ވ&oCj-CgL-F1d6i9xPmOL2, %Ju-cڠ<;B/HvBYI?e2VIēSY&M(xk xj v񽚔.Mu.ΞGFNÇy/p ]G|5=KJDmP3tQc0S^r]%Z^[m:+lV & R0}Iw/CT<\|]']:qMˣS&(Ȇ2DT*:/Mr<NCzY|$LdnxI`VsVmZxgPd:8Q<<NjJ-Xe-9AC/NGykQ 8yocIثwO_D=_iș].zZHbr"5$,q{8ԭ3YIaq} ^MGԦ0&w%ɣ࠰M#)H.n,5/5=55?!!u蹫I! endstream endobj 267 0 obj <>>> endobj 61 0 obj <>>>/Parent 39 0 R/MediaBox[0 0 612 792]>> endobj 269 0 obj <>stream x\K6ϯ۪"h @r|sx<;3,UE$$voU= "Ǘoo)TY\qtSRYnʛ͏mkg7k.~x~lyrv{o~׍`9ncm mC?c5[7r5b{^#&p¦=q\ǭ8PA?xl!( xO#qW'|u, I3 Zp6 /OPƉ٩+o&ķj\ \ g?Ы,G՚ԟpHS*b >KSx*TVP9 ?Yɫ*/;[K-`)Jy|M=Y mdC5fD(ؠc;*W6.T#XK坂;%Au/%8֑x̪φ64i˞i`i$v~YpBIUThkjXFdxY8Ck;û[ `fͦC !̂#:تǴ3$ +8ewc- B?w\KGgQmLfEeL ^<ĪS"p"w -ZnӒbX'ݑ!'J)eᖕGX#xU psrc3$9谤$iLR- \&% V5Y׳aؖ6䁂"-_JWfK_UosG;:t+ހ .kj)pQ ޚGcXz  aJ[ucͅyBX֙d=F&|:(jG^|2`^F@<=9U0]g:n0qh\"m^ L]zlSP"XEt>,\\:@&,"-b.m2[Q4-pNUMM pc A\KW+L^J&jaS?A )+*t?jghH"@l/M[IBS #Aqҿ`'$Xx/ 3]Й]e:^_ejFlyf8#V]ʬܗ_&2N(wlUd]Ѫ SYa潀j j٠j6UDV|)jժJ\L2,L".c/ k^QѬ]5&J53cדsXT3g.paY=(~)\4`ʑ%ܗHP)` $b۝i7C=ޡWN@ŽޟbS??6Z3i6Iw˧fSt|QRk7vrcH"; fvWʿiBU`s Jj:""?v윧2\V+dI}ð劋" b`J djEE*Fn5O+H+ -tt 0# ܻoyx {xlC$R|(0X],.7˚$B# VՍkRB &]K" 4T͆eXmf0z<7`f)o\^eD[r{E0ȗ݆qѴBԱ`Gg-C#WbU:GO|}4q G)災m Es; pYY/Ne7Y>#6?⫪DKFS #SG+F wMZ>^f~Н[t~$)قˠT,OU0*u{=lXԨ8QL ۏMg~;o闎Wk2]`.*W"c|o|6/`R>Q9C*X'+vۤx.=(H۲-z?4?0@,߻, O8A&96|-] W0(+dPvZ֔W*U-ZD0GW?? LV1ż^EԮ֧U}:z@Y- ^w`|\8|K>'oQsE>_][JEm:r7dn/-y}QY媯C}+`|X EyT >!/HddgC/T 󳌹^ZaET-' *O,Q 8y80TcOC^W$"D&' }PqD ӕ9-P@l)F3-Hu]3':c]8,m^9xģlHŸn{lQ M+̯ $'MfJYPnScI*`#r|ԙqk3ǎ0Iw =m(N@y [ΡߦY{Mtq2B#iv3Q|ge6RUDqS#?W " PigP ˁ]38M6^b J_̺''cx\h]~(;e3a7NP$Z NDoUm(%lkNYƑ/?IL}ȸqG̈&J8Bl&x3i(ؘ~-p4dɚ:|;0g:ŷqgĖ2TӪ(P"v*rғF`pbp'o%ؕIS)G0VH0B"G2[2i℉o7 )/^)eqYU# ^=К?-+cc9 2%CL(&m %{(wW4ň`~гedmimYW9v>DSR'~˅MѺRBedZ2_34a:< CKT@煿(9'B\%_q)ͷ))?D7(W^'ʊ]| h ;@9,1`sb+ 5YeDf#紘\p6oYK0H  /6ΧӞ$ƼG2K\{9\=@E̜O8qs,K#Pe6QWH'ar.DVWx}1m% opMu@ -i )  ˰ׯlZ߷wuSr9?r *\4 DA1 4yDV &[*f!V c6$Ġ[/ )Z E# +sd@*<5)VNǹ$C`P꯱<棌cIܦU9e^JfSi+ቄ ܇ڧp>'8tm teHR ;pIXN2Wb.}5mQŃB:Y`á_ALh?QXQF5 sۦxq{` 惪L2zicT`ٛcTgx+ިbt#uI>Bgt*nËDOx < QtW endstream endobj 268 0 obj <> endobj 272 0 obj <>/XObject<>>> endobj 273 0 obj <>/XObject<>>>/Parent 275 0 R/MediaBox[0 0 612 792]>> endobj 274 0 obj <>stream xڝ[ێȑ}ﯨG0'mnbXU)RMR.~DD&/d^##N\>5kL9gu6~hɬDT0("Õ1 g_A(4bdWuَfB?ΏfW#F)T MpսV0Mvo Nioʁ60&ni\_tsYM}ub$HXF|)Sݗohia6ˋ`O]/v_)%oA}u$!nJrjIN/GȐT8U-F}b_IM ]ۼNpOA{""qyf\I:д|FJ}r!ٌ.@[bq5/ǺkN]vʘO4jGwjϙI8ԭ(+K-fc",-Y@NLFO0V%sx27FMJ+sTRÈݹn, ∛v$'GoǮn|LRr,CY\L6%/ BF^XUA MQ/$P߄zи%i'$4$j>Ć3 $%e=XeZ_.M}ԫ1g_Xa(NcrIҭL>K/i-wt~TDW0 pmzյvAb I>@WV]k+i簂6AU/$VD!bIt{DamLP$$Բ΃0R_CGb~In +b-uaY%ʃi/Z|z-r!#[Kj Kw6jt -'@A"Mg@r$+GA"CNq3#S4WTW8qɭ>_MЛ6up#CQ&&O{DKNUҋZ}sAl!"W+C|/BWxTr ,n3*}kq[pqbqm%~E_餶DhEu0?|LN>kq .cn>g;7$']>hby=l>apB})<(Yj mG@?+T$ӈcbk ]PI[=]㨠>0+wDž&v>Mˋ8P6&)&Fߠ:5$[ bJĎA`޳k \@![vrg6η1hÌ\-6UcweWL2%D⡺tG=Nַ5i ދǃ.V|k%ٹƺ J7{"uęN~(9F" sFM\TmRUOXў48BZ+ JG>7Ad+m`돲8q`ץRC<_$-K􃉕u@#0'xb20Cd߱6-& H;:9k+UI*a 󰯝LJ53A^dV<{W3p}A.,ܘ:Voeq,\W+LpzY#` OrNGXyY0$F3wJS+m8ǡt"^inV!hscS8aBK/~50>N[Ac Z*8 (@+P΀D^{V/=sOUńMFuwz^s񥯔΅j^ dIç>%/65ΜL㢭F9 MXv:_Ʒ-0@:(>t)mC>^!\ean k/|<{|[D  ´.@' G X2w3X6`,ٿб( g@q-_>=1eJlLJ۾RK^|rMYqc8'C R*y<JX A40u<-ZޑΎ8΃@$*ހ\BTy($c%j*uR$5Mn $޶l`əeY;rN4ݽڥ"A ]r,$j aR$m ,ET5՜w,ī]jL2uG ^FrWEŕƲq0ag TE@j$KaUEJfPU0JSد D'8 N25M&5$\Ssx) tӠaB-sz!g 8aed T71ԐaƋ ,~GQ| ` vCEVȸibT܍Rf)p Ni}[i4%^Bbu>Z?I4.:whbaetF܆dAb 'dN^%\_T0yDQqg-Hj愍fGnZLC3@"_!F%` Z0V{X9esj1kӀ_g$v2_N%/N"oKۊzvZcu ܪH7SknqdZΪu!B?1oF^,Ԧ(Q=6s٦TuEv8n i2,w%nwf'DFIsNAzsjJ6&ËsVF#IL#lO5z9j:TȱzД|lPy08is%I8DPj Xb'!"S&6*et#*4e6J$;YA(m An#=( @ ͖Wro 3"%ِuˁ]!g>n`υ)|Ws}B<?]VaͣoY$ )`,E0nK.FozӘVgY@OKR΃01~蠱jkB3 鼀^LhYA&,~鱑QlYSz4WJ~2ĥj.HR2!N(4sv bO*PZBY<Ŵq͵7S6ҷ@~ڻ]n5[:\u~z/0~/?7 endstream endobj 275 0 obj <> endobj 266 0 obj <> endobj 235 0 obj [/ICCBased 286 0 R] endobj 234 0 obj [/ICCBased 287 0 R] endobj 239 0 obj <> endobj 238 0 obj <> endobj 236 0 obj <> endobj 237 0 obj <> endobj 263 0 obj <>stream x1 Om O@a 0`30 \ endstream endobj 240 0 obj <>stream xc`!0!@b endstream endobj 256 0 obj <>stream xc`!0!0!0!0B endstream endobj 249 0 obj <>stream x1 Om @a 0` 0` endstream endobj 248 0 obj <>stream xc`!0!0!0!0  endstream endobj 244 0 obj <>stream xc`!0!@b endstream endobj 265 0 obj <>stream x  Om( 0` 0` 0` 0` 0p50% endstream endobj 261 0 obj <>stream xc`!0!0!0!0B endstream endobj 260 0 obj <>stream xc`!0!0!0!0B endstream endobj 245 0 obj <>stream x1 Om @a 0` 0` endstream endobj 262 0 obj <>stream x1 Om O@a 0`30 \ endstream endobj 254 0 obj <>stream x  Om7@a 0`n endstream endobj 253 0 obj <>stream x  Om7@a 0`n endstream endobj 250 0 obj <>stream x1 Om @a 0` \  endstream endobj 241 0 obj <>stream x  Om( 0` 0` 0` 0` 0p50% endstream endobj 242 0 obj <>stream x  Om( 0` 0` 0` 0` 0p50% endstream endobj 243 0 obj <>stream x1 Om O@a 0`30 \ endstream endobj 251 0 obj <>stream xc`!0!  endstream endobj 264 0 obj <>stream x  Om( 0` 0` 0` 0` 0p50% endstream endobj 259 0 obj <>stream xc`!0!0B endstream endobj 255 0 obj <>stream xc`!0!0!@ - endstream endobj 246 0 obj <>stream x1 Om O@a 0`30 \ endstream endobj 257 0 obj <>stream xc`!0!0!0XB# endstream endobj 247 0 obj <>stream xc`!0C& endstream endobj 258 0 obj <>stream xc`!0!0!0!0B endstream endobj 252 0 obj <>stream x1 Om @a 0` 0` endstream endobj 285 0 obj (pq.graffle) endobj 284 0 obj (Stavros Papadopoulos\nSam Madden) endobj 283 0 obj (Mac OS X 10.11.4 Quartz PDFContext) endobj 281 0 obj (OmniGraffle 6.4.1) endobj 282 0 obj (D:20160509153720Z00'00') endobj 286 0 obj <>stream xU]hU>sg -?m 2VMIfS*tv6;df&O髠>Ƃ*6/ŒJ5 Z (}R;3dvA2Ý{woJ4~>9}JMiRFJ ƎoޥKg[c[кVc(ejD] <zo?9[)׀\@z`:ojԒ,;VlcgnFf'Lgjx𒡏0۴NcN{r6fr-ɚt%Po27^uNgqC$AZ<->ê9<2%+þCМd9zZ"{F?V|ȴ 숊 cy-..A/b?h/Cs&ԬcD,?rzxo9=k?RcU}y~xNt2ɥп7 X } p| htZ*%LX'R=fYAoP~קԀNց^_"ذ.M Bg3p4<F .h?c>1' k&%^?MOCuE|֝x2KQުY դ gyカ\= !F;AUm5 T LiVw7#NNma4f,T&4 endstream endobj 287 0 obj <>stream xU[U9 -Ct)Kݥ[kAd$L }*⋫IA-zRPVw"(>xA(E;d&Yje|oB%6sɨc:!Q,V=~B+[?O0W'lWo,rK%V%DjݴOM$65GŠ9 ,Bxx|/vPOTE"kJC{Gy77PہuȪu R,^Q 9G5L߮cD|x7pdYiSශX]SzI;߮oHR4;Y =rJEO^9՜gT%& r=)%[X3".b8zᇅJ>qn^\;O*fJbﵺ(r FNԎXɂHg ˍyO+-bUɠMR(GIZ'ir0w]̈́Ϣ*xšu]Be]w*BQ*؊S㧝ˍaa,Ϧ ))4;`g>w{|n Jˈjm*`Y,6<M=Ұ*&:z^=Xźp}(oZjeqRN֧z]U%tACͼ^Nm{Х%cycE[:3W? .-}*}%>."].J_KJK_͐{$2s%խטX9*oQyU)<%]lw͛or(usXY\O8͋7Xib : mשKoi1]D0 N }` **6?!'OZb+{'>}\IRu1Y-n6yqwS#smW<~h_x}qD+7w{BmͶ?#J{8(_?Z7xhV[|U endstream endobj 288 0 obj <> endobj 289 0 obj <> endobj 290 0 obj <> endobj 291 0 obj <> endobj 292 0 obj [/ICCBased 323 0 R] endobj 293 0 obj <>stream xc`f &v^Aa2 3q9E2+ ^IEU24?#&~q2'/ld6f3`^!3u endstream endobj 294 0 obj <>stream xc``dbDY]@R !+!Ά`sRA*nΏ*cċ`PDhadEYP7y3 endstream endobj 295 0 obj <>stream xc`9h &v Y( T'3&@MN_LJh ) h HH y endstream endobj 296 0 obj <>stream xc`C,02ƒO``?;3=LBr d`->@UN&v~QɁ<000qr,`eQz4 R? endstream endobj 297 0 obj <>stream x1 9QbDI.$}]{#_Sj! Pb` wC[LEA#PLAFb *b!0jRs)֡yɲ^-grwFԢu/%OSv(H'*@uO?" endstream endobj 298 0 obj <>stream xc``dbDICHR I 0 B BSF!!elg,EPASQ FN!I!N$[X88000r2!Q(dpz! endstream endobj 299 0 obj <>stream xc`!0#3++3#>&V! عX<;Dj<1 Q hf"%FV.^! uMQ>D;ZC2pq# Rq2!0 endstream endobj 300 0 obj <>stream xc`Ah FV^ I0+#D+:@UN]@BVh #/@4ag'FF&#FK} endstream endobj 301 0 obj <>stream xc`A FVn!Qq  T'*@IFj)#"@^ _Hh KL,,Df&X 4 endstream endobj 302 0 obj <>stream xK DCiUϪUcBӗNb`p(,k(*qRC\}JSf% Ee͜xgF endstream endobj 303 0 obj <>stream xc`f, &v>!20?3  ye2$ ^ 2PVgyaQ20?'/ Lld6Vf3`^!@ endstream endobj 304 0 obj <>stream x1 @Qh hѩM\xKǟJ5.F~1&a1r(%RoaH¼eak,X0v) endstream endobj 305 0 obj <>stream xc`"X98X`cd2ae)*Ne"']PRN@V f# ' /3, `GX3)z endstream endobj 306 0 obj <>stream xc`#E~fN>Aa 3@f)u}C $7<61p @(+y쒦~1 _ 6qšvO[M!qL²ʪeYa&q L\.V$掲.td`( endstream endobj 307 0 obj <>stream xc`!0#9fv..6f o1Q' dž'FVN! l dxQS3J&V*,^ڱia#ōK C9/Ȉ%8b<&VvV!Q玆2 endstream endobj 308 0 obj <>stream x= @+)#,к8$g`r2gx$$w @)}Dshua$R@ yc<'?e8/4&VbҎqr:VRHJݎ%`{w_K(W,;o |i0 endstream endobj 309 0 obj <>stream xc`~  Ll\d^.V&9,>stream xc` 02"kabaaB0rr1y9\~!n$0WLJf#>stream xc`!0$#3++ #60|">stream xc``dD1EY +0;?!Rb<%̼*b%"JLp%n|=eF9]- d{$yla`b`EĄ‡G; endstream endobj 313 0 obj <>stream xc` 022cdfeFaffgFV-͊,#F!t[xEx-*+Äp;B]R_visw[%$Q]kcFi5yA+ ;` يLh endstream endobj 314 0 obj <>stream xc`n< Ll|d>.V&1,<Ҳd)nldU~V;%B\03X9y,03FiD8B endstream endobj 315 0 obj <>stream x͏A0Eg2J[iA"&ZXĽJrYywZ^:kE\4[a6!k\-+q1<$wL,){A0Ϻ\66(}H9 endstream endobj 316 0 obj <>stream xc` 022"ibfeeFٸ٘L|p. ;L5#,/LWVOWXeR0a*% 3SH &p(`M1 endstream endobj 317 0 obj <>stream xc`$ # '/?񀏛 ]PBVh - %A4Pdched"Hs|C4`fp endstream endobj 318 0 obj <>stream xc`C, B_`@?;3ALBr daPRP*'s;Ԁ Vsٹyps2(=T@ endstream endobj 319 0 obj <>stream xz xTU9ڷ[ZRZԒPY!  T0"An!-#`Z< ch۱>Qu%ϩJD_O7o=?=[ٳ0b|!D/#կػ[̵¯or{| !+~KmJ0y`S.B];6:8}uv\7 c˿msFڱkw]qꞡk6^;Hfg0*DHnD b>Vb:@Й7M_".7/Zg2]jm6gهt 'КlAs9F>.EyvbvJ~J'$t?R8-N:oxW<xY} ,n +> zQ -q w}Qq#ъGT M ig~O>ȧZ~'&'?lRA&;'4ZgxE'pAO;>37}H=gO0O׎;ccw0})c~C37,;.SEz ^^`/~?}y>pxTn{t$Q G:=)n ?:P'h0FA#̻#xd[8i8 3?ܷTؿ/ܸ/JwŵhN>6Zn Ge!SI1Xl.P,2jP)Â0,ma86$`zhuzJg9aF/x%/#(L*$.w.K(ĹԂ^-uj%BwX:QgϢLzLUsBVebΌ{C8?NAo94QO;4@eiYw; d68R'suofȓp'"p]k\[mp vu μhmkF>aҙ?lsӪExnȮ=]ûv+](H?v!Ev ,Xz"ȏI\#7i;Chyl+_4A藜/=9B ܯs^%m"m'JUkvFГڅV4BsQ'ZG?Ah,f6"AWmk"Q7>5,8nBh ~}gFd>_DGЏϊ.ٷYv]eFƜaQ!!-*ʣp8DqZ`1&Aj*c8STE\>/Uo}a}d$׷'|o k=JGgPA[3,7wҶi[Dٲ)'ZiLonx\m l֖GѸV ;4ۛ0q ьTAq$a^:$(3ʖWȠxtj ]76 \֛a-nུpA6(BLKChj?tkZz:ؖ1G27wm[E=(f^{騏IRQqjh۶EB< ѡ47aWTOz=UI 5X  XX% hzz}bKM8WFMh#8L /"pw@\[DDz܈m <✠-.7 ͹LAK/bĸXi# G(t G2)WcJ͠V:WF ד?)`h^7(H^ :bxZ9.CNzzd0Ӓ^Jc@c1gG&Pgc# bV hBGPoTl=G%Fvqp`GSxu/0tFNͩT="#0}4+l˯5f`h'p+pJk N6,92Ba~|yCV˭Nʠ(Ysuoft5 ȷ'w:0n@d Bpa1>¹m_L»7eR ?=p0YSnQwu@]OEn KÝB n݉| 0o1ZAKؿN+ޟ!(DJt jsvӂ!W~Te ޑ$Ȃ Cࠛa1>Ŭes;ê%NsD{nPbHgh^S!3JqHzWhg0ara';<j}zPW'f.)0DJ8bi'I1J&'E*NN/DpyTlS2f`~5cԯ^z);!?˲CT0aӃk idlAC̊ڗu^vp[諂-L_)jjYbJk8:K/j qfUA<\ x6[e܍Y_jA%f Ӄ<@ϠO?&ԉh9DAPaux Z7e+"y^v@\1 vQ)5|Ox+JJƪmjgY3կkY|Ul[ޱu^h R_q\0[W=>wx>PZk|YɫWI9^}}h]^! K6! L. ˘f?c1ɄL}G,EYK^\lo) )ʥ[%!SH %C]{՛`!N@ ?.,l "̔*uPṬ*2u3775<''+ ͅeqr]7.^w~` c=ƍὣ #acKBDI`k%)X}:ѡl]P%y44 *h_:#*W6^)A&f?>AV+YGp؂`'A;{07d3 ߩJl QqВ !EqT@`d óM9 .ſ(KK]]2t R()II'L%a(6WPrٟymiF*Nwertx.ϊͥ${n_=ߛhp59L_k`YW1EN>[{niHS}GdȀ1,HAT&8RBFe8PK)+mʴ~yȲW0:A=2 0[zċDUL(zvQ|?dM{!YN\]Q6S PjPi(i4F򴘉(O,!XTԥb4;ʊ=tO-l{lUUV*.+ݷ.{ l {o^zD!q4hWHE<(me?<6Bn}&j"oatbr:u=_,9Zb9U0xyy$ 2#\ G5w.d,Xe`inZ"HC .;+ OXvzaGv¿0Y?5Bԟg FO#&>!&BQ7 F+E@XLF"/{^P'"TlS3 D!cNMh47$)rs<(;y4S H9J4`wW&p1Y9i: 5A)B6vvBKKbqtD~`y#!}┬1ɒ !MI2&&ITV:E FB5;N'W~o?O_OG"@«aG"??9A'lDlxv=?sIӎ:"ymm5a4П9PQY\3[~wmHXܮ+WKn苙niۍ6Zr;=b"Sjʃ 1̯QV,Z(0j6aU űffFC@v)U|b J eZ J)WP/1xYSx`%g<`(Ϛy'35|\ h>A7eѪjqj`%i_,1 RP)ח(*cBic@9,;l! _fty}Ϊ%y2"8fnVB1PݚUAdd(oɩ3 -1-9596KKBɧ 6l b)VUlmR)8n*z G .aJUym.'k|6ֹ+L*F=RgK@V<ݤ.XWk+Re'fnXaP]*p77y-W<ծh\E 鵮_9~!HĴljɅ.*s hgs"\AheFDƨ|=\Kr%]C.IOլR {//zTgcȊjK^>|Z*EQ9#2X^mIcYQSXie,#9u7;_=('b!أ[/9sTH,01>V~`P#^bUryC\7Ds憛75`ޘ3"9Y_&y%6oM.PO&MG.5#%#eClކܒ8Ԇ}۶ ڋ;x[ =[6Q_bCqauhÅ-Bq.jBj¨}=N~c&k<[&f|r9 ʏ7t0% +**cT?z詳 ꅜMT^I`sX hrbOl%I GUŐjǾą4ܹE˱WUW\/ ~ni+p$ov7u.[%Gdyހ#_[]ZF6y"De۩ܷDkdFשO6ŃcgKxC&q$/:[cvűc Ǣ9Y A sv=Y(m4_s\N"O%.xi)MmYp e~8_ih^!,uMP<3!Zd/6[vٽW~˗hBN:1nnv mK\7:0J6ބYo`W\KӱK﫫h.tۜBеz]1s)BMO$dϗg"HXՍ^6n@ϻ^į%r 2-E[Pi ڟ:27Ҧ$l".+k[Z {%a;[..A9L?h:?]WǟT(%)$ hP\)\"c$F)Y2:s.O*n4X(*libC m݋7&Ú:y#{ȽfCBIꞎ-< q`L6Gg[lk^4BKXyq(PeКY\(?PCe!cAl9*GrBŶb5G - c6%Qj4k90J I1A@َĈ9컩ޜ(pN9*SɊyQ܃KC~f8DC\AjtWE]gg߶MG.vCUW},kE qAh3+{V@NI7u#_QF''V7P;Lzs=vBi"'$QUZ]hvnȴ74_+aBd(iS aj$KGpR"=OY^W&)h8}4qjB`<  R@r-#~=\ЈtGԍ?!9>}3HH9 $$'}B_)=b}TjNY͆'߆<'Sۦ..ޚt+MGvxӪвhCpc-O45uh㺕GŸvOFV՗7ڬ-A|SGc *B.h9:oPݣ59f #vWzH5bTFTr5Toҁ3[!]3J dI.s s\e|9^'y=?ɓؕ@cZ5*4ʾ}jըʾb-/T5JZup^* 1Y;#_| > sG5JujTFi+Jr#|<`'r jꇪV%WQilR&]0, x؀ O"粍FPY7Ln٤E($/@gAK^W>wyР Zjڂ01X +ufD(J!Hc'NBvd& 'IL oBكƊF% ) \+Az*q,I(Rk-𼮨q򕞆}뵊)=_>^ts+J;ߒsh-FBK ј ~=[3U Ƭ_Z%I4YM(LJ5]9N2 Z%5h~AC<`WW~ `*I%x]nzPhu*UQK)ؒREfklҠvиKyz =^AmR=p EN+ ⥰5Yf7>\cptoЉ> RҘAŤ؎5݂Dk h QD!_1Ɋ3 (yA5 ⑝}@%B*<PS*X*n)6)0-W)ĕ4Ƒ5 :_55ݕ˱3w/*ȴ5;*_gZoꢈ֨ $>kmA) / j.Fh)|QXVOoW¯"WZ~B~@$N~!x]św\sց+Hy endstream endobj 320 0 obj <>stream xX{tՙ3XH%kFK%S䇄-rlGN$%r"'!`Bh$(6NK=dL8Zsh{6-(pNuߌdcr?vw}譻#7F£JCͷ2ٶ8[vdf$omu!e?gB aP=VUC]9GCy``y#7nu0#D|8{d'eDDR7гL1bъ$Tϫߎjl>0t͵B3_{-'Vi7wh<?P=0 ѣ~~=zupξD="G/w]]b)ԑSsS$ŧ`'ew&ɓF'.0G*Ϗ9^A30acsG< >-W{b6^y cxyqpZEzr0|GI{ ~+>Dӯ9{f3gWCzK0B^ {{gscgӜgnJIzΎhއ3E̘>8!LD3?~0QD=QBk'W=1gq|2zX g%h1摫ǰ@`؟4ƆpEZvXzcz{Uv8Uj{Ԧ.aUUmU&sXFSJ%XjIV R+$Rmi $1 VuL- (zx(bh'd/ F24]bPj]2uvqu) /\?8>NtpJ޿.>y |il|6MrT",]pq# g$=SEa6<4]fCS)Ol{x+<wmWa\ F]F6SEœgdd611rхtbW##Ns]x(+H _| $1Y7t OGf3'|K(@Zğ{N\&x>z͠(`ax#2z 4\ChmGPz G#h%aGOߠ.ԆGGK_'$~>DE9 vl+;WOAt3xz~7C4z}>GzEÞCAmF:4LĈ/c8Fބ}S!o߀MԠ# )t=~9 6}^OC}]mMtTml-fSРtZMV)yrT"8\WFc:MV5Qk}6c+4_dj)uږr'Bw90U0mI8[3S)bI\q"lx*?m̫t| 󁂾SX<OHtq!0nN ~)}RaN I9.`C)LimJ9i6=>)DC}ЂM 1 p n)(Z>sNFv~Dd…[`VėJ|D"QXb2+ U@ӅJlA56~/}1C^&t  w2p 20{;*ϫ B+$"A %@ƺVAfBATS8 be678iJWW8dUa)v~_rrxa2D2&!TWWf8U{Ƚ L a~=xd-sd M`8Up/ne9:0"<tm;Ω XXsʣtm{Wijmm2=n'G,Hռd|A8<RXդ^sf圶=Np@&`͜ t3"ˑNN15'<{خq&YDA;):;0əX @1p ln 0<#6 I8 hb+U$Bc8JH]]HޛMQi*>:q!k]pXHl0t0CimfnUq0j`´H&~'C{&3lrA~Wh0 `tF8ȺŝŽl-y=yfIfg)hP"\gA)ȵu ]ӈ&Hza2X+0keDW`|oל@(IۓIYAOѪh!^ixT:mxgϷ4^oLsw WK>hIΖ evG29rqhphʻj#6UVMǓXiRڔuʐOEyRfݏnr7Ś7zO٦7zFQ8 ӆK줅 |:[JJ&5Ӄ@eX+ k3uޤK$%,XNK<[­kW[.ꩠUMai&Z5L9zO[E7n2X JQA=⹆vWNS; 5ϿMܕ;ayzJқuS '$iej9%JPW^®{̾Ѿ.1'.xYj3)^(J'AaϜy'~y7}Y Oy'!Yb2^{Nmt}= }:xw vy2B*iZ3 oVZ8A e;Ռ8vvD; :a ;^d%,uGcTh(>mD#9.z} pZ- a7?,yYv)5TWh:PArDȻ]쎮6^3tS_ bh[ scוֹ(-LM]+ 0R ث7ZS=hPnsl4 ;37xPST)J Y-1^$^ !%FT0 \$JE* v]h9r\M_{I~6W+8+YY{f`m[0U\ڊ#NMMld-.^7u!ֽw94/ДS?opU+fczb읍[wUo6y6k!P/KdWLԩvljX3l)v'yIo9PEZ֢>(D}V}>ߦߣ@vFG|/Eu;ݝTH0f&EZN@ a6klZ0ڥhެHt.?h;-=#D>#D+O"/ ?X~e4:^a7>{Vmd#x?X`Їz (*dZ_o*-u6[=,PDQ -NcQTS(47 I(;6l+]q!l !bBY} }~^7:t=%Vp `=g&1Y~xp0J\+jd^޶] ϛ0닷^uTEE XWf=\kYd_bO8^WJZN֐!8VgܻK3y˗q~KW^ D h7kZ<ֳ1#%AS >}NDZf4UsF#m$S~}HD{&u^-n;.#w_lp>7&F*.kJI{^ :έ|iWv3kB]{A=\4u,߬Qj+ZQ9k4Mi=& T#EٌKWI46]Dʓ R܁Y@/D9saP^.U]v.YӊF,dRTN)Rq{<R-+SZ۝!7lg)=E,7M+cWj݁z`ahWlڵ[VZ o'Ru 0xE[r%|9 ^ ξx.g[lifb endstream endobj 321 0 obj <>stream xX T\y}.˲, ܻ\޻ؗEw8RTۊcp("zծ%iS%\icY][#iZŵ}|d-9'If04=#c5^1ޑvz~z<|w ľhp|^v %#y > { =to iO"[ .:z`G{1xtpk?']~?㮇?b82ڦDyw.4/fJ*8[u*u,*RxK]KX_bP)f ˝N˧>sZGOZ'4 O>S&<BSFR}$<>Y(! ou-ٱlw {xT56:2(WG2JQVΕfsR g^~z#7ݖmOϴf #BȶU Aǖ &` 6'sM\5a gYF&0kL$ &VZfA[gj%oiVkm3&جָԔ!ߍTeTug2}ۺ"3ANd 8r9KRѨ;_oQGj %GeϞ{P7QJ=S?Js?BLcg@ٞIv[[ZVۃj7JASt,zQ̭iJ}2Uxck^%5'LιBs ^{{zS/u^O9㹦̆窫uD*P[m,)]AJ햬l{[]&C VXp_|]e&s6.0FGv̞ߥglg󟯑*^mյ7+YəSNξaMխuy|OV!_Y~or}dEt2l\qee#zm7vK}MG' mkVow[x%ܸ]_ E%O~ދ-m[_aXp\3˱]Flv\q&m|dJs}:\fKǹUQϛeR@lY;6" 6"շol sdƍoZٺ>n3;fGl}?ŖƍUg+hol8e;a^/ϡ @",ZQ^c%V>Y7[7cLJNIf>x*۳N!0`Jjz=8d1&ӚRk"4P+WW4Huxel.l'!;1늼#̽Okңoxα056#/ʲ& g6ׯs1l<_/h~](IB--EIͨ/MU".+D}29{CVn%gfgYJfM~[,tWW9+:eP<]-acoFg<_wȥ4fsFmnFаfKdȒ(|(թ>jg k8"+oGS;q.Nڥ|׮:mƬQ{fK|uK[_02OFϥuU)&5+59}--b*4b=Қվ;|bJ5夋JNkq(v϶T^.nH|Q;`3l/.dғVְCk;40|ߎ=ûQ?CE endstream endobj 322 0 obj <>stream xXyp=$mˇU|;.^dll*qJ؎\0.B5 M@`Ѓa u2$r69OR߮d3 ?h|}zݞ|0BBow+hWb.owH?EtCso,lx |wF|m kEm8脶!9Qc=.pd^o/䟘GtB=aY%y+|atQ~G =0`@Iɥjƅe^8Jg_y̘ _"j_`6@,}8EG7N\QJ=7w uq ԹAcRpY'[L1eyLwAh| kp@vC}MO'YN`91QDž e-'3e$YOXdri6y3wZeoy,#y8DViQKWKKf!3-d_ XY-SX 2eob@Mv1ΒfL'Sٙ9q܏ wXKʞC* 2*vdWfڳk,Lߨ_g3V}~!`4k6 z/(#X t^gIǃ 1aGŋ0^G*GQd<8R-qȨ< >(`eqy4=/ .i-MRq70I{VW[=o@owGgszlNu$pGj$6ܦROR!`nUͽ6ahu6>GyKF͵92׊o\ BIw\KTs:u%\Mb6xg{CN^-HY RUB$xұ !0A,Qip._l mڙ`Y) &> cvo{1wˁ 52*YhߑpbiL$uڕ$7F70Sm0S:~8@60<|rVQA-?DmV'ރ=؇6IkC6̦V B`NK1F}AbV~Im8p":\v|a3.j#v.]þcerZ9I`EN_!KЎ%XKq3{p!֝6aޤJ餐~Wf״;`"UQ9Y Z:RSMI'O32u<2yN[G2<aWe) RhIPy4S$<ŐDP y&H8l ]?谹iXk5!GJ!HREo[qHpD1j` fy Tbs8ddcNoTWr:RV7H5EsK'lmlEhwKbnr$CJZqsn(1 /P!-*zTɻ2. 7PsLtCCl嶶@y1E;|%s'ɚ7hZ_b̟i}~!֟zh"`<%YFNSoihF8FZf;3KbfX۫a$6G(9O?*9E{0N"M\6h7ٍA5 #|ӪL6u{=%fr^ YFYO:Uܤy2Dչ#ns˨la첥dSKA󓐟G+qw 4sZ+B@T k lpNH3sS5 !"t#Tib=Lٌ:WK;R%]sICTn7yF2׵9O s>9);Pc6lVi(H ,#ܨ Tru4͚f*SG*H4NʶHCx-Bx,O%LG!\J9Cx(nɛIيYcA1&LG!\A9;UgWBωMIVSs4knµcAx+((T?B7F$S n? ®1!d: ᅔ[ExQa1Ua .yu?%>ʙ`OT#rb C& SW lx"XMnIb?pg}NB߂ 2p F uU&@C _CGdZ ? endstream endobj 323 0 obj <>stream xwTSϽ7" %z ;HQIP&vDF)VdTG"cE b PQDE݌k 5ޚYg}׺PtX4X\XffGD=HƳ.d,P&s"7C$ E6<~&S2)212 "įl+ɘ&Y4Pޚ%ᣌ\%g|eTI(L0_&l2E9r9hxgIbטifSb1+MxL 0oE%YmhYh~S=zU&ϞAYl/$ZUm@O ޜl^ ' lsk.+7oʿ9V;?#I3eE妧KD d9i,UQ h A1vjpԁzN6p\W p G@ K0ށiABZyCAP8C@&*CP=#t] 4}a ٰ;GDxJ>,_“@FXDBX$!k"EHqaYbVabJ0՘cVL6f3bձX'?v 6-V``[a;p~\2n5׌ &x*sb|! ߏƿ' Zk! $l$T4QOt"y\b)AI&NI$R$)TIj"]&=&!:dGrY@^O$ _%?P(&OJEBN9J@y@yCR nXZOD}J}/G3ɭk{%Oחw_.'_!JQ@SVF=IEbbbb5Q%O@%!BӥyҸM:e0G7ӓ e%e[(R0`3R46i^)*n*|"fLUo՝mO0j&jajj.ϧwϝ_4갺zj=U45nɚ4ǴhZ ZZ^0Tf%9->ݫ=cXgN].[7A\SwBOK/X/_Q>QG[ `Aaac#*Z;8cq>[&IIMST`ϴ kh&45ǢYYF֠9<|y+ =X_,,S-,Y)YXmĚk]c}džjcΦ浭-v};]N"&1=xtv(}'{'IߝY) Σ -rqr.d._xpUەZM׍vm=+KGǔ ^WWbj>:>>>v}/avO8 FV> 2 u/_$\BCv< 5 ]s.,4&yUx~xw-bEDCĻHGKwFGEGME{EEKX,YFZ ={$vrK .3\rϮ_Yq*©L_wד+]eD]cIIIOAu_䩔)3ѩiB%a+]3='/40CiU@ёL(sYfLH$%Y jgGeQn~5f5wugv5k֮\۹Nw]m mHFˍenQQ`hBBQ-[lllfjۗ"^bO%ܒY}WwvwXbY^Ю]WVa[q`id2JjGէ{׿m>PkAma꺿g_DHGGu;776ƱqoC{P38!9 ҝˁ^r۽Ug9];}}_~imp㭎}]/}.{^=}^?z8hc' O*?f`ϳgC/Oϩ+FFGGόzˌㅿ)ѫ~wgbk?Jި9mdwi獵ޫ?cǑOO?w| x&mf endstream endobj 324 0 obj <>>> endobj 276 0 obj <>>>/Parent 275 0 R/MediaBox[0 0 612 792]>> endobj 325 0 obj <>stream xڭ[Ys8~G*FE=d]kiv7 Rq"yXV YEMFl DGMy"竻7ohDZoRIUaoz^mU/mӷu/sxyD6['şNTmsT +x+ڛ{&w4m3jN,=N]lv%2XP: p(|UP.7J#(+6 M*TB^QU{Um^nC\3 7 zvLLRQ?]5վ˱Ԟ΃ߚOA1;M`I84m9^)T?kgNyw2}{-a]Di4ItYnЪcQ,0\Pu;oE\Ëp T{8Z]}7]C>оSzl߃@?]l׎dIg,!%df0j vOWI!O~0b&6螘wnNݏM:!>0#&Yǫ'~x~Z! ƶUtpCUg20;3tlq{QK(M]#se9.U=g^fw3ҙz4Һz#n@ĵ@QK% D7 ,r|%_ق[@8DTVC,h#N$!pd`/sC㵛_ҡrj`$NrDa#W6b(0J (dnCaDti.~g'||4%hG+4õ+\B"k3vUOk|C#߮EȔ[hGQX<5e/0z`r/nW _~y㛏/WU{O~}ל! 8 b0Y[TXIc'B1JH.K# . yDum<+Հd( 3m`ۛ)Hfk۾7~NV`˲*"z^/>Η@W }B6DėVbU,>ER+];68AQnHn,: "}ic|xP& F:9F$ITM1dB`H+2A&jfoЙy:FKP eo4bN.a QAf7w9j< D@bQF"ln tVD0 ힽ/1( 2Ul#]pU~mQՀ-f\J:vyR!C[)3`Zpf r1yC ia/lݎC>78kZ'}8'TkN%rPhdd.@iM/hhbYN0OTefhA(,) tTeh7ÙAaPl;e/, twO{Nrf< X\`h ˁ'q n{;*`G1̜jx[v$'E6F>HsR.yiU0p, 2ݍ:+9Wv!Pe7rf-7dtWa X.~cm7k:k:B$^t5N( yM6" ڕG@lLm#3K?@*I#+Z|l0h5ٽׇ/fZ3u$,.7w/!;eEцj2ct($יDڡnW +IX4kD_l`g~p.fr;'KʲkN* d>e/Va9>A X\3U >}{04hۚJf372E) S,9'G2 Vt A3~p E$ uEE>vj4fvHbØ52!%Dw-cvWP7G .khl U]عd16R&瓔Y*AY'!ABoޖm*"Xeױuo@g=8^c='_lrkEOmo%ѝHBzൈ4P9 ( tηUCN z ]|AreݡqflWo`Tumj~`%XqY/V 'z#9}0btYiAH'(w]eKZ$p޷Kqziǂh31PL H3Ȃ" v@g$\/E~_0M>w,J%/d_V^^b]#+ՉDD=[8jjl.p҅F2Z "BS1mj[躂2_s!"v(DgcoD׆wj+1dA]y swCµA) D#8 /tX1\x֣/?,k0g>Ô0 Dr㔚4C3"LdU S`d<x8PUPOV܏͹b=KzayӺST'~&)?ӛ_nqK'ie".# ȏD")i5Nh=qNrKxmY-#ǺfK^:,@؅R̀N?$k○(r Zt<N飑xB͉\gU0vĊ R ٗf~Y o3,O1KXWEVlcزtƩ^~ӪɒМ[á$^ ꕟC6.f)њ*`G"Ә>܀MT,x)%.x :$*+GWpZȜX)g],]=#VW9y)`V9#2_K U&`Q˯' Sy6޽(f@5-Ī*aC@e_8z/qsȿ@Bs_7z6a 6,5r+|'&!*2K(Vf(DltJġ^6@?͞yC3!H% n FJV?-\YfN=YlJi$Ѽ =w[yY\0g[NC( L~v7q&% 3 ΒNq R3#LuD MkȬEdnsjceFa9yse%n/\se|f$J2'u` wk5}L_{ݨeӲ%bŚxcHJĬLb3qVv.2r| JźDJ R޿#N+v)w!y+6.䣱A?`)yFI]jvb[ &s;_[lW Kt({lh]I_c \IA :#aW88 رH햟2($&)g?c~We,5c Zrь-'OBhTe,+c:̋ 2fL f:lI,jZۉM E ژOj0=P endstream endobj 326 0 obj <>/Font<>>>/PTEX.InfoDict 329 0 R/BBox[0 0 360 302]/Length 3922>>stream xZn߯EM58Lb{aȲ ˀ|}Ωkq@ ܮ~US-᫺7 ۷1o\to_/&}%.|w\9Pּ)uϑO?^>^ğo/nq¶IneӴG) jn//?.y&vqt>?ztGwsy9'.a7OcVl6_.w чW:Weռb:ϼZa/ͯ}rz{Sp~Jӟ<-m>RX/-gm^el/8c8c̽=wڻ-{<DZJg{K,cPm 8ӻ5&x^ݬ_W)=W㷷P'ĺDڞ ۧo^|J٣- 0_w}w# CjelĬ\\>Pb8ؔq{޻t^S79˯D)Z.8qHKJ._u/|(f*{康2 @vro?/+ dwo/%[TpV }*R9AYDv*)S&IHH9&F@TI5i@$ {{epQ[;pUܓ]'`yNa &d2pMFqR, ]IC TY1[w1)wI8) `&;K5"Voͥ4D (Ѩ ,֢=E=q|>4Q0q7-v&'Z5iq+֭Ъkm`z ]ơ@n4t.SLFu`a=4S3\ (j@ #jx{r, 4R+* m L0u2Ȑ@FZT󝶨ޤFZ=9ta?{8 )2_j&mR="S H7X}l d:\ 2r[.ӫd`>ojpԂ!A7@'^݂XTL qQ!1A"XX܈6֩֨c :Wfc :j?C} "C},SC} tejq8t#S}L,(kĚ#S}L, DZ0%ckpJ0goy|ً.Ć̝ՠ/k)_G ONJuhn|E)͓:;_FK\\7P5(l!a?Ylс4k BtOi2TKȨPQÑA5l: FbVmAd,̲f|ZTyETm,K(ah?x,#bka 0oQTz(Y4yHNd)R Rf՗@U0 P0 bmR;M{\h= \0 vʼQr?W2s61~9dehEVIF|4ͦ f'FƯLˌ8pN]WLXlaӴ+%z([` E%0u@\@[d|6&`7,9 =v5`~x*)$ƛ_Wa L/?M~A$ {pLv/v~I P:'a5YwA)p8g?E`+d\nZ J -iK"'$|Rzr2' B )}a#]`kN i7`p d18bI`ckV=J`8Zi83 9-Bi'ٕ1(4'!). ۓ. cԅ…ˆ}}PAav2V`k9U82-r`%S %u``x{26`5-U nL,d0['2k %>/Font<>>>/PTEX.InfoDict 333 0 R/BBox[0 0 360 302]/Length 3375>>stream xYK bnJԃ6H_.{m(;vkqCJ35!Iw[ou_[b.efۏpKGvK7ũRR9iYnۯwﱷżݿv&aPڮvp&oظic6?ߞj_>;T,2ZywjE}v 6C0 -TCa5ݐezq0\0m\׵eZq]KÎii㒻G01c=Cѱwhvy[{1w9o9qYԔᴚi7fWKXZ6eᴜi9n.VzâK[ᴚi7n՗e~%!e1VpOXH1L%G:-4Obﻤ/b`Q6|8776,s:9%fQVXE9-6u!q <><Rԅm0t%!qZzYK?XK׵Z<%^+1zSq pjv' yCѾ/|Es}to%?isT{~bޱ}d1s͋?OIY6qdyii Tc7yIdX_-KȽ.m,rs;,Z y^~vmɢ!.|>O!/yn*g>cqѽ]z\pV.xio2;ǯ$ArttjܳJI"{2oٲg@kS \v2LQ&Q2U4y*H *`1溣Q \c* ( Kh9-g){U&{8Ixo :j\1qhAHfZ2tk 0:[ Y{qWArBWvQ#gݭL>&X|:7g#cnVցF*0qJPPuHts{f]{ anTvd@;Rwc#(ˁVF(NBx µbw$IX;s&m{R΃z(ϡpʎ;9AҖ\9ODR#y3E`BDs@#zeĉl:2ZXi</kGjYiyh: ([ѺVD0&*MF[ yG(Zd&[@N}w ™yBl8jBD'Jo 6>inǞkvt>7$6qf 7HAfj>gn"Rw>ifP4|e>.* jg쀎A1)S~pi#ma$`,f+d"c#a *+Qo8l6^S5;SL1@d`ea$!;J K&QEpث}z[A+~:1o(.?uсGļ[ cC1g {4M~[bG"z\0֢&'`\+Fajk<{Kɒ| js#|K]9"W ZlWD#@TDťv;ƀ*R2$i*F-- I _5A=ɏA$;u 0`U`*RxE00Qɝ`H;SC023uX  ֳ#hHtLxqbw{ `œۗHYR>QSIpDz+K0e+qݣFLx62^BI6-7$_-6F7密r4*L&u7suX+ :U(;Dq6Ťg{-< F8jjDYF[FSpbH26Te0B< 4J${OBRX gK>8bQBH@@1Lp%a\Q'd+(_At"#1]M ur<{$tPJՍ{UKRgܻ?4(f|tF;ZB@ZV Ex]?w.1,9Q0h;\RS.QpQş7WW@>eToA&R%yLa͉xaET$"GLUl)Pp pLmaUeeY {$ 6PU $1z*%"\-zP-3bv e4JYeD lUI .@KVlbM- kmn 6D۵:=a:dԾhm-U#Zc͉SOӘy$j,%iUgƎ}U=eGAE^ G׷g'dQm^aجMg{V0p,E9%AG;c/c!q<,=x@ƞHI+qK4 gX\R a7s78UuP_pFY 0GgUҋD{3c u6 k$?ҽ?KCx%E>D Zx烙?hgY/Z/x-S#;߹>-PÇٜuT8](7pQ0pY?{P/#." Fkg| B* ܠ =<XܒkpDz_|\)1Y/z?Ye&Ez>5^ߥUxNi9Dӣ ^C/.Yހ%G{~n!͟<Pr9NN2 EͳHoI * g|Gٿ<= endstream endobj 334 0 obj <>/Font<>>>/PTEX.InfoDict 337 0 R/BBox[0 0 360 302]/Length 1876>>stream xXn7 W .Q~l]E&Yy9]I~IQ{Sx uHQҽYqe}iy^YiS__aI61z 9~Z.16Ig3r8y^_O.wVM,_#g_vkYoGO^J @GCT(&X։%Vg)Y N,:KRh"xUyz<3܇=Upo N-MSk ~u؉e;TPޯs!|L$w1sw ~jw>Y 158U EJb*SAe*wL12} ڑz>qe@gg :Nˀg 0!(3 e A-1^ã::vsm99; (3_OBgB'j"1q0tFGB零T].LʀlӀlhcws3%7(ofxgM>-7qT#C@֍A2P (3LcpNs9QTVHNԱdem99;'1{ZN>'ML'%3J e AQ C(CrRSTUWϙs꥙xy#(ws8l tONtym,#5z! aoebOЍC7L}f 9l]&Zt?=gaӞ;^<񓔳35#wᧅFzkgUW߮/}zb.Ԉ Mej7TmTj]RÇϞnbЈtG .X^$7/%{߳RDJw/_JhUwɿ\pG{Ok9\?LvR1M՜zHoJ~t}{wocj]u+}_#Hq|O?a$'[A5>њios?}r(fxmvpu`Q]@idiiO&J׾լ07L)>G f#} xQ=[rw@2Q*8}ȲHU2Y&{IZ$Ēv. DI^Vnry?9Pl A1W 0jc= TBO-ȑTAl]dL:Dљ|P$G->S駣: JSbHrJkCXA&TLɞ} +m8(mO5Sp eJ?W5:h%O2rq4&p[%E&@CC1GiQ%0nl u }'y9T.3_ ImPSmMX$kGh.h4f#Hٮ>\ف &e-vYE}wXRS,j9:Hd= Y(B&ԞImP_AѪƏ,۾Cxͭuxa#~ߺXZue 1ip zaq,czm8D@?>/Font<>>>/PTEX.InfoDict 341 0 R/BBox[0 0 360 302]/Length 2571>>stream xYˎW4,$U|n d!b{Ȳ@#@V~N&kfW4zSFEC-dlkhT7hL&~75Jgg\w|yyOp_ћ:85*jm1' oFQ \S"i ߽fD^oFe[Y94zSFEC-$qШnИnixVoj5N _ug\ff?7g EY`raEf4O@洶HxICAcE[Qj8^QãzQ#_s2-ܚ#¬e9 6GDM:nPĺEDHC#A#EDknzSԨqj}&71ܨutõܨe~nM,7&5iA! Q(MRƩܔ{"8W5n?\2Ӊ~"2s/{6)4:FQw~vq|5O֍_+ss_gsGvS(Q.޺ˉh2s_IKm8+CJ6vfvb>R!A=R~/b \qRGeiNk8@.;L 꺊cb|^6Nw% OIگqDl"e(c3&Eoݖ␢%j}P8zaן\<[XuC;3a H-d,jF*x̌bv sd|8jY)(zT ۑ B$`3DG֏23A*(VhrgEcEvfdRgH?D#N/udd8=fIaсYvuPpBHXsQ6Q<%I¨bVL~Oسj5H3u#Q`%YsA`Θs=A=XԤNMŋ|lZ(8aNؙ4\V1}ZyUP\xG'nv7/0 X>܈O^^ &<+0y=T ,!'Q+I*,LRyIT>2,LRa"`Tr $0I>LR,~!ǘDR%_K/JX3n6JXi%RWD, G ª SjTh V)1Z@29U?2r W+V^vp)d)rx,S)r)N dI &lUVV}ѰJv^YE$aU{ZYE$aU{]YE$aUx*" *nxªɑShe«A(Si(Pe9U!q {ZEr<1Zj@FH [ EZE* %[rh1r*T` '`2F(0Yi>/XObject<>>> endobj 277 0 obj <>/XObject<>>>/Parent 275 0 R/MediaBox[0 0 612 792]>> endobj 343 0 obj <>stream xڽ;k㶑W*WMy&|şLl]6dwv|@Im ʳ_dRS5l4@4xW_㫏?OUijWOL2GVw]tuyj;*P4 J07:^|uY7;u{{٢XޮU nw )ş\:[ÆdĸaW1h1a',{XOpwucׇDWu]}86ޟvrU2 'Q >vf㳳Cm)JC'm~Sv™}gVm׺?mM7ͫr1 6qϗ=*hz 3<#ja: 0K=[E0xORFb܏*7Him%KH(͂gdr![Ɩۊ`x͉.~$10 XTm[`3o:68`xFUY;هAi(B_Dž;B9Ǜ.C# [K:Q+dPZ;k^˔tFݤ2j~L%?SZN(1~ .5UiũbCO#Se6&Q"}nl!ˏr+ŀ4I mH;Q"qΕR'W}TG t}Q\X\mל-IplOj "+ ZF6ę8Qїp_䶠|WK忨SPF(N+0034'C2b=E^-@F (rAu;Qk؊A)Fw`䧯 6L˿G^'|\X^@ NjNڲamU%&vaIBߦ9|r#2n}}dqEatu1D[OŻcˀB)]fОX˅Vt<~ hN 9fv L yx2V?З`;qcaMf2a(0( W|^ >#޼yx1( " [ r !!CwX:puTE`U-VhQoD8M K:zYs<0TũY?lϸU wx`c4`@j=Cb .23 f&3sO}ý̷|h`. 6$-O :|W&%*"X3,ׁH˄&ߐNDO/T9 D\9ߤ JyPQG 8 UމVB,W] |HɅ>^A{WڠOR;Q Du'QP|mK~:J$`=]UI8݉2R[~btZ*#P;6h}aܖ{#7<6x w١z^zۙjqk^M|mrG-z5 6 рgDc$-–=vf r"?ܥqZ.sQ" x]MEd'ͷa<0d {ܒVfȊȺa_S#q^?9\bZ 'D֎B_#=tvZ]}yC,dJbr` XeG < 33{nC)@8!I[?JqiwTC23oBfz¹ͪՐ gOƷM9h zJP)Z1\"/‘d95Q01i>XbNg}T,웞oB\wE@r!e)wOFZt@*p;;ۈ`P7#J`Ӡ߶gXqG I<Mc#-)ԍZؓebzْ~"j=D Hpy.&9e_tWf^Ʈ-hA)@I9E9J E|vOshAo:J؝-Sr-RKuqx0 # RS 'bmʗRTa8YG(dmX2D,=QՀ"-xt{@!ߦK}j4@@w&ٌy($2,bBHJa&1ɭ+Vb@2ǂATsJPkl~@OQ ijwF?<)cS;s A"Tݺm/vh SDcXirAˏ/ڼ&T{I\.ӡ2 ^AC`,V^=@7v^kjđY,BYh@"k5( "}1$.Q\HпJ"mYGh)))\rm8Zh2~ߎc;r%T6FI$2Ct.{,b-< &S()31^N$  cmʱ 8ԾE2cWm+Eԧ2^y+ ~u<칍8`\1EoƳSW(O=ɚri>ߨS(xnc:pq4JtoTݛ"06Wd,\. ni j6ݡ'~*VksKj9R#!;IIϋڒ`&P6~^$}5Yٙ`M!+|W"^g!uLFoh"g-$t1Y.!s`Pb";)B6~8 avN2TaEGȌLI mY<7QbHq"M"g@RV'"[ĢdXravbUmtᣉ5$ 'LSqJ_M6\GE`Ǿۂ"I.etW]%L3l+bi!QԀHA2&2 ؑ ` Yr_|SJXZqLv+hHsS>F9%?E |2b+?18&Ğc?-?J a5kWoEմ8&'1k*uDt: /yDv栗OE!J _+9o,j@9cGp b xU)6>A/as6vԎ mtuz=s|<(+Vu7ׅK(Hקv\]ùŔm2+It`1em<8E8Ptl ʗҢ'fbTM|0+pyx ܝxI2k1,Hy6w'`nJAQe+do~ziǯ{w]w~λ٤5F'QӕM@&v*2'?1{}%`ÿYaa*XWfM[xM`aA|i] ӹ}~WwPg#lM'_Y`n ^I{i71021`B8fDۥs(^mjC 7!cC W"(`NoY!">p eէAL\`8.iX"GKe $ HK=RBoz-=˦b;nY.eϫN7% S0sYP 54gd^eǁ\eb&l~pMAIL ӫ8z**J>и|V;ktJ"Ău {Ef#R;GDoHUADZfIDY 4x0[~p_,],T\ /<,Wɭ& 03F롊 e΍DC ,f۫ŗ51X_1n"_ÝkuoO$MQMŨ֯a7c$6LCs}mwai닊TApkQ;t zIMUwܔ˽LUo9v]Z豸'˱DYFQj[rn_!@'Gღ}2,yFy1%cS` H!i(TxCԱrc-`Iވos66L BlZ7-<#t+Ó,Ԁ2w"ʥUa҅T>1(WX)iEK^JRuX>d bcM,)ǵ44{i|Mei\$ѥV̸Fm3D-EPROr5&\5 S{g geFKAݸT'CVbINx:[d/9FZ+!ŕ()c~.˞ōmp)_+6P DUb{pH`j8|JW~Ki I?F=piyҩ6_xGkr7])dマW;Z"SzX(EY=[EKTj)ugG+S`] k_=ć`+l2'6J~*EBqi4S4/r_샶~UAOn5kxV#"$"Q,҉4Ztߚ`cp52pYhb6SY(Z%6ˀӌeszVEq^b8,$q!c iJKƂ@缠r^ [; (~,~@8j=5q޴7,6™=&MX2`9\QOo_aQPY=e4J{V< ;I𵺽;6t?PVK endstream endobj 329 0 obj <> endobj 327 0 obj <> endobj 328 0 obj <> endobj 344 0 obj <> endobj 345 0 obj <>stream xTiPv /26A@D(f@=22,*qxAp:Q. ,Q @@p&DPzH߯{9; $I}ؤXYbt.eHv vIM{,Vd@ 2+/7p>G -.ް;eOZb|LbgkRs?ɒGflil/Q3#]. HJ_w!v !ʎ$XbeaJEXC|H|DD0p%B6&/G$ b6aD9C&I|BAH55I 51T~$ř}=ܕLс G/SxVXh;>]xe*ӆo:+ D2%W5%yGӄ w, [-Ŧ=mkJE*<~}`RhJ 7 ;A-6 `[MQ5ĒJB"Gxp-fVxFZ \~Reng؋*1WӀˍ6?:BnxD.{%NLIf.m?Y3 xy]*'G#f\2bE)Ϸԙn+ݡ J:s:7Ms_c+TΉ_xl9 wpF8Aߚ b9Sv$wrWXbga@)˂}0iQ}gkXWAkC9fpZ:֥+;t^iBdK2~M|7{pb{@0pBlxS <oG`BQcxavBa0lc\9l&V톣qpn?P*RLxqa2i82q, gF2OD;Y@tM!uEW1e m&t Y9x^G0ptTؓv&Uߴ~7QQU\@ˑgR_Z7cgKJj]mF〵n/ۊ-V$jC1=h<`F6{'ATz/+T*8L;[z uҠϙl{g*.8&zɯ*0VD:V">+,rmw:oq+lWl~hk)y)bx>w7)Sv/Zv3h`X"$vډ:/VEo-nUByJ|AO] ڋg}I$TWBf~^"/.K;m(+a+)#)00`ACOt endstream endobj 333 0 obj <> endobj 331 0 obj <> endobj 332 0 obj <> endobj 346 0 obj <> endobj 347 0 obj <>stream xTXL?̜9lj!f%#f*Q"kR)߾Ůg#C%ʮBiE6FnhTʸ\=Ͻgb}|y~$@$9HRgRVɓu$Q<}b h |X\8|x77i3R^SK?Se$Jr*uZFY6S) W%UgI__w ĩ12JUJDZ2s)ip&f_Hb11p%yD |D(F $6aOC #A G/1L={m/C^|9 *./(DT>uC8BGGӠԖEZ6%Yk :DD'T EM7?_ uRa~X釢5y"2~l2طfr}h!q&(k=hB퓑.gՅ1?"P&M D%\w:)2W&'c]%?g`?E 53bb}%, h YYXNEai7 wOF/L(0@A@ 9 ơ|2}tdF-1O]qBr ă@A.+5Xr?jL-;uo%u7\9dx` E5"!9PK{Ń( +JVvCa=W|gډԍGNK$ I=KNJ~z%-ZV7oÃ6ߵzsr$/~mQ]ŧ1t0ǂAi6ddWmABUZ{X(yqB?rȯ>f׍-sJjBJ8<1`? -Lr艑O`8_6_]P V rllfPM)g;<.`ӡ++(6IЊSiWsn?&c6DtWC-dp3~4n+׸iˬOuꤸ12}zĒ&kb1BTrzيFsN\x_ӏjݴ-Ow@af<9[+BwS?j?n}DX#[wuUr1hUXŻe V-|u%0>{aX]Jhg ]B?sslyƃ-\q!+=x[78L>k`3 '3(LBq@GR=عDa獹k쳭ZXLBA3-- J hN<z=Yer#L0Z<4q9"ڿ*NXh?k8/_>aG[k_KW1 zxpg,=EB5&s6Qՠk@ʻo-\2oWė/=&C\g.CPQQϧk~'|VWB$`+۰V_}jJAF+W׀9H zM񊌚[~XGWɚ{5U74&_R"u?sHނ,?[CU?u[_[1+z&a$szW^zpm#ڜؕ !~b xf3(Ȁ{T[IvH:*2LRsh=_7(\[ў%}hQAډ5M% R7xTrfbP pï|' @z pa962Rx[q$0| #ӗ$y0MjDU e4Óڅ^ }Cu/{D駐C޾ܨ0^p@S>ya^pQR 墙#IW50B,4dT͙]'{|s'bڔPJ-D @G͡? UK$0XL(N<* IC, 9Q MAyɘ~{;*bOh!TRDu}Ʀ{oD endstream endobj 348 0 obj <>/Font<>>>/PTEX.InfoDict 351 0 R/BBox[0 0 360 302]/Length 5062>>stream xZˎ-qܟ8K ~li1`i. 4E+Yu833f1ݧݕ?Ou{ Ǐ#nxKO\g号'Aqu9'o?ݢ]>ݿwX~{lGu8z~?/^\6w~nx;Քk[G1QvVZm7FٝZM>j=5Ӳ5ϻ&/,f]x^M#Rh >[4"Yoרոƭ׭bj u CZ}zo{fwf罾I1my"RoYòhD 5k}ѷaD_I'v|ZmOVG555ϻ}R<)!7aZ4ta khHެְFX|2jݬnw/Z}utse9I>e3k"r?R9O`jKagcQ92sJ[Y7jT.\p52 `q5',г05qp 4^?rd.1!rIkk0kbL1bJ<݌PH3谚SS(SK)+*},+Y>%cul( $`*\c@'7C\-"$$Ht@*اfAHr!r ѠMӁFf.D~XYZ7Q-cM& 1bީu3)ZC9ՇuJSJ\$6(- vwKFH''԰lF:Kpp+ï?Snˆ)v'}rTz7L(N$}yHA]⛬/&fy-A|3<w y2ތ* ots!lg߀$b1CxLm.Pf,CaJE>L'QLsUS ];Xψ=`N*s&4h=V@0Èt.}`Pz`}Qa `"%ꙦB0GY{.ZTP,qXDXRX3HJ'&LKl➕B˰?PafӀ{5ҜbLK d.ڹМR,X(R!Ӭ۴1IdIo(mL [/Z4jU鸄i#-Z* ` :Ҥ2Mٸ*y`e2X 23.ͦa,%fi&gXb&SxKi,l8)C^hN1_ ) ,82,?}BL_lb%UmOK;Q^а'0d4rax3IzuWONm ^"# ^T#6i*mJ'R')a_cbaxNw*vdAMǑ3^IVMGzR02s(@dUMFQV\0Pa Zm_.}'r-PdQTclj\7[E( lSh R;U+0LZ[WIVULS7Z $ϰ+<-^(rn1,^hdABS5+dxМU /tq N) -2P0DI# =`53 H m bFz <} Q Cl-b}-bi m2W(ꍸO:c'ԩJF>A`XA[ ^QFF̒*xfhrh8ZP$б |mpDSJ?Q I2bU"5JЪ:y5"ʑXyM3 @#7ENUQVdՖ3N{Afu ( S*-jXYnRҒE@R]*}MOIbqa"`l*汣"t? +LƯ^i·SNM5zGk,Cm F稿?E@ endstream endobj 352 0 obj <>/Font<>>>/PTEX.InfoDict 355 0 R/BBox[0 0 360 302]/Length 4119>>stream xZn,W4%@⬮l/9Ͷ4C 5Xlg~.[O__n&a7>&{,ۯ7}[ y˹oo h5)%9tjq|0O7Goo]٤TOcsqw׾'ExRWZAFl4TZYdԪeղrrZ }<@sC]@׾-{ d6;sфeeղrr$İ;S]B >?dYƁZ豻 Jy|AÆ9:.UXOՁkuL). ]ϥ/ prG"o&68w"/q! y }s.t4\ʩǁkuL5]6t-K_@y>?M۬SDߌ5?~n_o;3vR~Yp}(jpi_|~nqMƗIP:}ís9IM9S F"٧~ns|ZûJۇL uTz|OóK}ajd{[0}dMpJPe̫Ҩ+ҁ&[ @uhݣ RSޔ|sCu* XT'dﺻ4dʵ* `ۢ5 V5&J `JSf0KEb]zݛ0ˈ(lxܙ<. ?XYH,in-O4Iq>x ?: >@zG=uI[j\k:A-YPBܔ#x*2_S;lu TSOa2)ɫoAǰLO{""N[} 0xb\ IGu`[ s5\ޚsÌd:lo 5&uх-j 5gBr* I+ >T)2 Y\G#!EQ'[NE%TPcE[JEfۭC"OeR*mf`'ʮ`,u8WUK'O<֩zw]ՒUENByWGDzGJwA!$EH~;7t5!jO} jԏ$g=/ÀC\#0߾ٟf!Os~BPD^D5HYV&l<"r~e}ዄ,a$DؔޡÈ3t84)1wߟ7i;رRIK7Y)ƥ_jstLf`XzЃ I@&u<[Cw_ca'Jףe #dvdzI~%IΝ39w+'wRz"}f:^=an[ڊ}E]}\ z$9KZkvH zVt zRgrHbo&NpWrsh 蠳T>T济N! u 5Q> C cXoQاc 1L6occ^bdv,[E/TnGQ DrcX]D 4+"ExLYYPz;Z,1LBu+l rMb^PѰ,א%Xq'F6a] 08g {M c*%p+Og yI`. $Y'zÕ`#:dC9!_`AZ$h/4V;۔.i,%wwԋ;B3xnO:7@#mϼ-J~bsXx-ʁ`Jf)%䙶EI I5efm6;*M*{jM6Bt hɁV%vnh%XrD"EFի5QLKͶ`h$q y-ҊND] DHR5*Q&J,3RH@#5#؜HLȶ:ޠ8mc^oj;`5y1X'C9@w4^cpR4uqoԡCּPXUmOHT?auN=GkO rb~C/ֺQrʋ?nDbaHh ;)Q#?4 -0u?T4 R? !7y{aC !m?2)}u1ٿ@r4ʶNZ۞3Ȯ$(Ѿ9s*8b2 V/]&+:E\enF|!͋P^FK]o(-+J'T^ 1DU\;-"l)G@* x'6o4<#k(U 8qNߢ5Ae)Agbh#ئ!C2J䝃nI-B4oNXf5c;=̓3V˵sJav[#ޏL頨+A]y :X{޳ݻ O4x h17'ۗԼVY̡E0* Ұ|WUeZYF0wT<Rú)ȱpa( >YC@ɾh(.Z +~kz7^F-.VȟȮHc#Q UnK$ m-%>/Font<>>>/PTEX.InfoDict 359 0 R/BBox[0 0 360 302]/Length 3657>>stream xZM% _@rX#pGy5$H=퓽uc`"n7{#XUd-q 7ׯ㯗oŰ}=}E~)!o9"eD}|wT)R2uuZq~7KgX[b\lqKaOMޤ*OkXjݵomJ__Օ~/r6+tVjhjeii岻8\#xb7Cp۾ML+Qh.E.U 9vpR҆nBt~Q;^oy~*iq8ch;&ӕczÆ87\jiCt[T-s[-]r|)ML8p#>\#Pް/΅ eKL6&1wnJ?|yF7ڝYB!_0R;/" L)SQ9t+K~I*&1' 16ar,^M&Oq3{$jC?-8p I̢mhW&6vL\jS&819g9NLy^I.{x&HgIԢtZp&p EOna0G0,^Mkvy^6<169=6up,nR~bߗ):_m:8^=G/}{|\4nNﱔ޽g"L%I7ɧ9Yҁy_֟rZ^o߾Oßui[5}kEHdFO^L=zw,tA"\fgӎUꢞ;~M_VM?t8LOIGNztz~@U/J^$EqϚHLhYyJc< Px9Ē̓8q! Bhv&|@i"XvTS+E&\= W^}Ϛ3ĥ :4nl ?H?xܙcއn~̺N$tJ4<ݥ< ٠'7 .~tZ|br~{Q6;ࡌTDBWsVX!0Ղ(ǐܶ32C"X&'؆Htj |0ՅrHu ' f#[2P B%}4 YiNU< Q͊F w`o/c{:ܠ5z <:Cg<,pOABwd @9`JʪiXP/14cVټ 0AVP*J\a[FhG -|*dӴuѻnQ tL Ψ%¼{xx*d5Esh쒸uXDl=NEo<1y,[!- g.)Yb吃qf#S2-p HZALPϐJ gcV _ :p &f!lRnigȄd) tWf }hXӳ|5H #Nֳ̈́b|!>Io>qg#}ۓIFm*gsGg]gʽ!,F YOi^bׅLO4/PLP %U3LPyqgyP&ˋ`pBJ-:ˋH Ff 3ME]^y=nV3=:=,E7x{ܬ&{GY۽Gx"z/*JX5HY#W'5xТ۲dcN($Ul$:%B8V]Z8Fv]H<ЁbR$E 0\ܒu!oY^yFU&@4i\sB lpr )*4Vg:V@" lU]ŷÇ)J|;AH=x ɤv)@(GNy0;Nm6x'FZPD|&ƍ@fZ"Mݓ!Hf3f/h.pzWjf Ê hhaDh5ρf"% jN4fɂ&ח3)A68:EaCcoEe*ҸԮ2-e`@`omה0eH!j LBUWKV QS>U}I!|̔r|XaȋE/BH5J!&N$dj{+竡I*UXrU5./A |xVS,%TgYIɩ.k<Eع$\5J|} KW,YɊ/"K,3(`־Ɂe\q1dc%>hŒIZ5̀K&aIJ/^'pI[dcIJYĒY[5,8CƱ,+r,פ4ѱdKvc:$V fGzސ><,@2u:lۧ _H|Tl*&A3}e" QK͘qd#ћ:PlAYjZ@:9N\O6;-eۧbMA]kUy+3a`G atҪ}2oȈz頜V,G J 6،"b"yF8 Jf M_6B3"L>J)RĘ7!XۣYmհbܵ dY n6XkmrDu( BJvO(dd J6}4(=":%5q({ȳP2C1(٧&CYp#YS %ej=e'ryB9b=TB/&#C=$!Hqr4k-PFJ,X"E'k?"ȊOb9 aiڟl<ݱ)&V+Y%4lb3ͺ F̙ݲ][I Q+r[]tO%AV|Zw1e.9JtW-M2OX㟋ۑ endstream endobj 360 0 obj <>/Font<>>>/PTEX.InfoDict 363 0 R/BBox[0 0 360 302]/Length 4871>>stream x[Ɋ%ݿH阇 ^U`T2݂v>]. TFƐ{Oﶰ-?oڷ-kT>d?j([)c{{uFy|s{L琚^m13e4ַooIo_j>m+^>jc,efygҖ.}wqfru ^ʻ=~EcEڲy8+iִyGM{ef)3#|dsƉ5%Wȫ4N̞9 >G{t 뷴y̲c~D#J}:\(@$)6|s2Hc4lu 1EYeغLI5悡=,lu^MaoM`RiKL{dF =:32_fqLjLx.}w0ɹ9\՘Oä]1iqd6|sk:g3ghi=dԺH][wM>rwJ\Rs/&~HwA=w:T^9pȖ@zcѾ-z=/QJ }ݚkw0ێ?&v` ߁6x;Ʊʑɣ}]bj_αw4v>F'kW|dB%d엟:CF^/;bH{;̮_?O{co 7吏?n_}>!Ji@,O??V9 6 d|o8E昃\,Blr^[m,e;uy zt|&dZFfG``NC1>f_|496h:ҁ8,SV a"/X0T û{m?ݕ{}pkV|WnmL"8?Dj3^SÉȻadS$6ŗ,\2V dh d,Ӹw^ȯx꽃΀&%-[54R"'lBϚ(r{(clz6$ݱlgrJT#ߟ#GRpߟ2H4KQJypbY o^]p&({n램XDH${u!tFZPy!d쥬YCx֡-:rOg5t$D"v= 8ZV|B}_) endstream endobj 364 0 obj <>/XObject<>>> endobj 278 0 obj <>/XObject<>>>/Parent 275 0 R/MediaBox[0 0 612 792]>> endobj 366 0 obj <>stream xڭ;َȑd&/nOۻwEJPc_︒*ch2#2Vw?.(8ZĻ_?S^Vԟ p̺BF)닎G __ݧע}l}_:6/Y T>+:o حe;p9-GnP'QTť]Gt; ӻ v|]~yw~IJf]r ֻͻn kf>@ahwaISC4jj~3GwI'.RͥܢJZ*G]˗bmp:A0>OC׉'tկy*n.sqq^hQЩ;emt Y8 mfL'eU ]C7]ϽP"< WJK;~% ' ]r#;S*&nhcڂa[}>QvU̳zekhXր&8&}{@,?3&31׽XU|Ì&&4[.#{,rQ&o[xQhV+],Oa )@ 8vE˴ԗMMP坛׶t wLk5 YRduIJzcQ—v$lE3=hvoMc-D1&f6QҞ[JKTpUr"\{qّwd$C$)_TөG*]ཿx `!Fޓk}iЗ% -'>< 0Zt(h(HsSӻYe=l`+ϫc~Y0v2 :Œлd}~DC4/.DM٘QG2Y2GT1 ̈'~ 3Q8m.T஼i@zy b~SU[Npbo "iH{lBhJm-=""d82+& ]f$Kٳi(wƶf*Iϸ[4`t4 67/sy4i 2֟IA5Х`€ xQp }8mQ0UUML_A=|: ~jYȬyDg?_ƷN !~D* yMX[k:RXe,9DEz0bG>f@ N6e4+Tә_4JN[}{ȚQHN{l>VΘQz19΀q%(a oՂ =gg@7C4(Mx_#YXynX֏vIJ'8Sts8cA8D-beG-zG  r۩` tdԥhBrEKwc R3嗅LQ)9e'vb= m0qXSIɬh9qId.6AmqWLPcT"Rk,d+vi1\3hRJYbb3-/ʪ"Źf[> "H_ 0c_D[bp/r/=?u8lSx&E\@X!f.)ksT<6IC;<ς$ސ+=i3rɮGMR<*Gžu*\-n^{NصrLp}+Hq䱟OS:d?F9S!IN~PY)7T_ GX=k'dJ;HNQM7JD`;Lljx,f7SMM7cE'\<8۟ j/Ar'\WTA413v% 5EM'3`h Ƣ9J&QLX$U=pnr*&_mn:2s-I& N bt|S`Mo!cզb%p n|cƣt<@7̃pIuV6De[p: vʔA3mZ!(d.+{=z 6̛j̶BÓ{(v־=얱SYINqm4KS>i=sQD,dcSY JI^4/ 3$""!bTOl?Ɍ7ƕM8VXT"!OFu"Rni)Bn(@5)@ Ȓy)/#h$J%|w/A E85FZ`Z41qD^BҰȞlpaXsR6;lXb0*Y!KX :2t,9yɵ;`T`LQ&uձ. H`}yGb&3[KtXю7╊Hė"Ak!3/439e* d#Ν+m mn:Luj|[茷RX8XrJhMqsWѽPW2`4\˟qpxlխ׊ Jv6z K/* /WV&ZXрsׄ, ս-E_tY* Q/pi!,9`K<(\&8l[x'/yJp%:L{@ER'9y@Z>D'RY,w^(́4𞊺h' pmYK"O]:&[s}QLšhV,kh`5^̂rvZ{㬉GljlAfv$ڝ`N4=M|T/G*lA_JQ S`νsc3IpN3ngнdAP.8is[ߴ? Nquj";̀xqunsxڴRʑN05jtȑWmUG .&E%/$(FHeġo7n)7s oAWPMMauWN.AUIɸ=\b}=e樓j p&ZR֫ wS3ZKW饢5=t3*;X#w2^b *]wVI{UKZk2GǬ=( X{լ뭚u^J2+Up*!؟KZ'VM2$H=w?Xd&IbNE^Z|q?2HR;K5kjE+US!v\(eь2ķ2>A1ҮΗ@>Ak[SݬHrZOa5JM5Uy: V<%Dk tKO2Kj?1f<ˀS[R8x[<)+ȄuI' Ifka|#zܓrA'c;Cڢx32Oq,dDY&S„H,(KLaTLɚJph m3T"Y:Œe\8RX8XYH$!\ Ma zYI ih [t8o^6:`5gTmH&IFj!VvK.2h1\Rs)[yjF ?]@h@Rգ-]I\  T4;E> endobj 337 0 obj <> endobj 335 0 obj <> endobj 336 0 obj <> endobj 369 0 obj <> endobj 370 0 obj <>stream xUyTgfF(!` KT.0q9PG@W <@0"hD0舊2$1+5[=wCvׯ:~j06"H eK)qɅArVFL©ú™Dfؔ@dj\f5;maL#"Bqpp\5m{FJRe~f$Z_iri"$E9+:,.5:T%?A㗩[ M 8u6BkYaK%<p"CxK|b&BbaJ BDK’ cBH&HLu HATӍˍ[R%(L( )Π3L:$ uHG4B6Dk)z;4tں+f=oqfDK-O֜kK E n> y@7ʂ|!G(>_-`oprhMcw9ׂSB٫-Z{ưU+Neu51Sx9АZ'gg9'Js} pw?k9+DAA 9A%};:Cf=!8x=5B אZ!4x 9k>\Ks=c0 {\ g66A1 wKIrzl8-*cҒ3'_øQGj-䡘:9=+3fZcתթEbsR=#&ushM JHgpRnd(9;o^IDldqREn pǜx=20z1z iTp/z+78YieIiRtI_:ZtvUei5( b{uJͤ_Fް^-݊+GŤ1|g(%_` <]ؒIm4x" 1X/d5Xޅ@A`ɄS#ض{"<ah ǸJXEBq%!x_] `j{V?\&GfT(v%Z!XLqAG׀ASĦ5Yj q"!K_ƉX}ш.j--dNeh0W3}Tf&ԱjonyeތYC OW0#W(+v1}wB}QPy[)`/vɨ?Fǂ& qP)vkxC_IZxjKW+~~1 GVJDC_l|8̗<3x*?R~jMĨ6..O!-;|{I/{כ^nS CN448 +OkH'm`N?c͒ ?Pv5~ Wxy&R*/g낮v3^ L"uRI!,@ _3x;Բuu`Տ#wڇSŇQB U$]zQxOapdd[)1JOZH¤ &* (Wv~2uA58;F~S(> endobj 339 0 obj <> endobj 340 0 obj <> endobj 371 0 obj <> endobj 372 0 obj <>stream xT{Tg1h∊&)RTR@D(b o|UG@[OQkAb҂U@A*"FKWֵų;{Μ|wpA*K5K.- hb9Bϝ7qնiI 2WE-;dV'%$f,j[Zޒ. MN*2UiR Ywj X2),\3w|n ɖ̲'5,b.N#7%;"‚$kBLL&Xb aC4BHyj'oȑq3ǹ#$ pU/ԁJƍ:'jLQ]Lͬ˾",VW󵶽"9Pb 1M ~ _$U$=SIО;R )mFaZu97'/j'5ZZ}4R.d pؼIuzzg%HS?^S~zAnf(1+'`ȏql[k[CIcu2tgۙڲ+ 1rey튲g'N-;r0R 17q޸q\_ "nx![$ 5#*Lf;@l0UpGYx;+olƛ|oVVn8 U\prOǫ!HN\?׀޻+> 0AYvƩ q]֜twTбǩH}"yѵK=Zzk_Gf?É.Hm/=K;-;$ j?Vz1VDP풦U :̌U=$uYNGP zXFA :p$e< )|Lۺ˧YnNH<,>k~GU/9gD*vH Jtw\0y R>)X(jA5q1 /dB]ȗzi{o_忇:a0gˍg;+wckl% $WKkOSF eUf867 *f{\k^{$¡aK,\}__s335o-0^T" VTlD? (aoRc D0xUޢ_N;foE]5|Oiz4gx._)Xu==?f2+zӧJMFyN[T>>^o1ͯƣ/iT\֢0~S?q.cn?tcC7C!ϘF:Dˋ=V]U4g($gm>XfpP o>b"A 7M endstream endobj 351 0 obj <> endobj 349 0 obj <> endobj 350 0 obj <> endobj 373 0 obj <> endobj 374 0 obj <>stream xUkPWf%HIP翦sw:AY>ٱ9sv'&&GDfJm9-wvqu3'K""6"$"&|X—XO,'W!BY!aDs BH|@<$,y- }"ErJV/T+``Z[L`A!ʔr>Qc)IG t7cϜerGJ%)M0U _%8rY;;!BBYp-Bo?`v԰3HP"J,l/)ݻ_P R36]lu rd :T85bqKװ^a{qlrt!oƚ<%7$-X):*+Ak26Z"lQU퍏_ X^Q QUS^$lIA.q` h躹KzN|6h$CR(8[I(P :3(yn/b`8nb/ H ah^BW;9 ^@J[Ir ֺs6Hmε+A4l|hB-R & DPXD8D/P3u4X=š8d-3$vʾ<ћ >jXm(I3'O}NefQ(!c'# {AʠG JrR-ai%b\-)cnb`JqWwl^X5`Dy.,{k|Qɉ/N|̸PxvFX 9gZiIoyʹM 14s؈%b8#:fBw$rK%61_~hP ɏY14J/ ,ÆE2JBgnA Q%:L^'?Ϣ]9y "&(fSqı: ^2sUNn3i?/h77xx>d%U5nw?*5Hn>_Dkri2V,NC QIŮJf]z_-~j:5mF10Typ7o '9XܡJ*^i/b#`v*~뻚]6M \I JX4E[QK3v˰?k0?+o}\jF[Oe]-8v=hzcGfgr~mrtՈ!r֌P1D(ϱi٧HSs4Rr)iky ;G7sLA PR~bi0䁢O]>?Tp|[Ci @*IW`2Y|˩zty#Mv7ͮ$|÷?5f,)a:&?;*{?+g$:y~i2>]8f.Sžw!ܼ}19@gtݦ@ݐ[La+ʼ1B'?}[0K## `7i0Bx9pKlܤ\2 pE"q`m5Nc_=Bs0}L$5-x %{`~THp cq"$D!3{ =,{BA^Ikb@A%$Hal`` {/aj#-lhn}Bgl86.c\l31YPbuQTe΢9߬s_;oʽwh9 L I޽3NjX]ݕkg:3sڈ67a땟a\&6;p:o{j.7msd]\۫/|WTg4M N]_ ?ׅ`a˵nh*Ryh꩛Qn]S(I\[/ =0 0 ^,CܧA4І8> rRƪ헢e-_l(&D XhhW D\G) S|ed4vh6A &'d endstream endobj 355 0 obj <> endobj 353 0 obj <> endobj 354 0 obj <> endobj 375 0 obj <> endobj 376 0 obj <>stream xT{TW!ɨLr*"J1" ECx+Zl^}}D+u}࢑#ҍhWwsn8ڳۿ̜|~;NMUZ]8O;=t"t*XL!grxp]*&QbQ^f rr +\-W>Jzm~q|~"eRf~2QUYS,HBKJg <ҋ jLͦeT4TOSΔr\)S)7Ĕk7uzNst}6;pdyEfxq c.1Vv){mC6YvX\{cL {]9x  j9Ww4tlֹ/Ga$mKi[J 嗯TT"{c1/ KKUH1 l؟L(n·-Va= gYP %D r gY;;`~CI\y{:-Pf"X'e qfɿ aU_BuM0|eיclm@3I$ R'<*!lYȆe&U]6FV l/ i\ ;yԻeGc޳mZ3YB y78?5[-*NQKOk?yq7QH׻}(pns]M{uX${6uӪdԱog-wxd8wTfIB\w=X dQ(}KF,P"gTv gXFd"w5q㮱A#2B xr?oODV˿;F0A`,g/#YH0"Ai$,XECeׁ'D D! ʿ/4@'O,=>L7 AÐNRhe~vںb?$ %2(ֿEݚpR0pI"͙@JÓHQ5(*]x|J !>0DZ80p)z偠͢k kAJ_ݕۥ {U55MnG\ĈߋQ]CC6>$Sn?vŊ!m`^f,LN^p6$Vf>+p>|1ӆr[*dwO1~; > +"~"d+쨡ұk37Hݮh}{顡W@/$|x j ydOd%ÓB 30w "؝d 7|S9#!7CgJz 2JdQk Xr@MYc8B ~+ƙu{@86ꄖH2Amq|8K2ąL&H$CؘVKrl#008'[?Ml` ?žV5q<w:O~< endstream endobj 359 0 obj <> endobj 357 0 obj <> endobj 358 0 obj <> endobj 377 0 obj <> endobj 378 0 obj <>stream xU{XW!df*qg*Ey@ QAXm}Ժ - -"(JE##b㣕uݞw׽n~3|s{)GQmFN[~Z !nc=(GE95D:QIK@08'X?r b%1K'y{]!'#-]/&_Ay&7#-S>iktL u\y&m69wKE(JH GWk2b:I~SN{PBʝAMM&RqjCQ Rj5rQ.Nʕ#%BF5ROhZEg6:4ƉfE?sR&ֈ1 &ɆsX{ˆrg;ֶNUrU(m%PJZ~FZ%ſ>l/%+:a6]-nY`qnfeUuY%7ԣn$ F)g?Gc,& Nl;:F]v~mKS~^g%uI3Ce lUێ Mik 6%+D ߋo[pA Ae4c 0^x=do1C`>txm<~f[QN0m ƞZ}fGf[AB|v6> J**B9TyWP̴jAiJbӣy͟mlG%Ca`dgKieqٌ/#?<*G|cYW$urHw=~RΩ1{E޶ə%-G(W&me?ܵeFĥm-~qTlp]mo=.Nn "Wm A=ԋłleа?V: \aG!rpa`x,)Zq`(6[(&"hзAh6cMUf\DbMx< <Kъckݗ"#ENu5{G 91^NzxJ2.1ȭ]oLtE-""?eQo*V4t#zsd0=Kw(fJSke-ߺ@_WgL_I} >]yB]љ"ԍnj+[u{k(Q|,qpRe5@y -ĉUOA:VĮe&a&@uno`/:(LADp^g"p@" d`P*c5ñÁ8/_$ߤ k= OPۊz Y-79kw6{Ežn69pdeŗ#nxT)rrUr/"MYٲsfυON*-9U:{43hUofވ^Vd* /WN󏋟tv1ũGTD #dnݛ^?LuSuGw>+{05{ήDcЗo*yd{ 7نe8Wlʃ W+NQ;UTgM.㠈vYkg0x]d%UQ]skN<)=+;WѪT(1@AUʘweuiEU endstream endobj 363 0 obj <> endobj 361 0 obj <> endobj 362 0 obj <> endobj 379 0 obj <> endobj 380 0 obj <>stream xUyT1h'MRRR@dS(Rd}@ZOAkABAiʢBEK+g{' Ι|߽{$$*UJM"br$7e7Up鐩X49tf,'"Y Kn 6poi!$"Ǩ3gr~Culfٛ/eʬ4 %[ڐViBS6f"Ҳd䍪̿l Yo2,M`PBR;Μ:c|قӦrbxx$K,# 9A#BPb41"l[BL'XbaGDb!$D<2NߐCruM`#H x0PxZh-j(1U@]'|;f>EPi͵ -~?Ek zg:\ uOn)%_=_!zj_f6A(4@]PU0Pwѥ/kk/рťsw*϶ɹPfȓl0i..6%q詑b<g)H $8DR,wޣa#'C8#!n<9h°C'3^5_9w _9kˀkx,Wg]"SKfI1Q=-vlϱ5;UVTa}| "$TNuZd*59'#|x{fU~D'nAL)'Z3nN\#Mj3:ZҚgP[d]?pUM^Ĥm=vJ 7hc@3f}B7&&Kwdz?#ѷbi ݼo j8^6ɧQ4*.(? djRU"FlR]>xTȯ_KT$7W;b!ރ2p0H[6abD>q`ת:"]'`4֦>Y|y&kl,KApP%/\r Fk6\z/3:Pۭ^͖ZC&cïlo..8[h.}&pC/ŷ #G;Qoҫr{8zn ?gJ!CcַI(|JV6e/Sҫ>>/f~tOPT[َnێM֗̃_i_9+_2wm s^Ye0ݯy[XD3/Ֆ I.-,+;L/ܛ{7#Na_-#-I iM ac;qwhay E,V+xMd iIC4Έ ƶ NCH6=IRsXy bq^zL_ ?b W L1+h&sr-L0m9pYhkU5^N}mYWjjV@sQ*14S=uӊ#O z> NY ܅]'q`zA-'h5l1U/j=gV\=R~b%__Б2Oa`J 3M&O[g Pe'ʮ F߰fIfuViվzp"'|X3>r?4\hL 2h$UÜ~[<6O=m:{*b W۝'}dEцo~TuIO{OpxJbzbE pC|y^^[Yr%%|E{uoC/@ /2}RK>/Font<>>>/PTEX.InfoDict 384 0 R/BBox[0 0 360 302]/Length 3148>>stream xK% W #sAi@,{ 8sҝqUT(5ﷴ˖wߴOoe?p]o|+)o%e{Yr<)Xs}9g&ўܾ~j&y{x} dT ve{x}!_>LJۋ'4;S&Nau}q.+֔jb+\?\q~w繫is\}/m˹coჳ}5i]G^6s^8'OϜYLz?W5 100%]D{E,hik>\3gגwg9^[p<\-}@ǟqrjë_1 '-gҶDHx<vo?}%<S˞V&o_'iKO1G~^ IMܔ:hkϿyqސV_௝7ޏ`}~ӛ_4k[~efޗ9dp⣻W/+Þ]jdW&}8wGL+_m>lxz7jx}7ӞFq}g,JPɪ{਌\v'8y1NN]r l mϑ׻7YNBkI &o8J, ŷ ͝(7qhȾm0g۵9i^_::)óOYz>flK4:͇}UEz4Zzkt%_kK8̣ g֏-Ik |0ixb-I# [oT@D5 `=oH]p˔b[1ݭq3 [S7HAp Vɛ-7S޶|a cK {U{KVu "4jh C=6rC F{qoem a=Uͼ%Ve FfZq.M(~+)hˀ7NXn>LmdZXnSAN.su{ ٗ7u׎-a_I1@C^ [8voy8fC~%-airؒmv ˒93[z1OXs~9mƛ)ċaB@vF |BLwn䅘bDMYArJ"1+b"2]3@|ٖW #e͂Mwi"c+a:-b:0^W^a[Cz\P}ALGۘ/ @<R$1o+b`ofv_3giE Vʆ1c vVD"Y"TLbP3ce b04.~1F.2P] ,4Ίؚ1*: b}šVԺ;(t_[ b,ee 5=c\|M< Kj: 3KGڔO-3b~-v2`8kI5GND6\b5L> Q>+HPɛ6(ak@s9#G$NrF 4D$Nfk$ b QC\# ,Y˙|DBof)FPFq{;b7JQP=F0T,_]Ҍi4T 6es=Shhn{z&`hM=QбpM:&\Τ#vUM%QX`or1{q3h3}0Ci/5^B^3Pi409G2ee†`Ԏ]N_7IU qShO ~kX[dJ݃ 8ysG _qj]yJ#у4l {a]V!@̀hH=4]P>(at#7iPdwIi:0Hd-跃4ErrfP49(j0D.EM(I9T-yrƘel ;U8;8c̲P6jUuL, E^N(mES0cMsiCI*9.Ja;,^p  G VyO.Tyquw+aK+{WiB; Ͻ\o.+c˛}΂ sʊ̼b"pو{搥 .C k1d5Ya С^(챧\qL1,|ڼkUO"W=_% (mYdǬz$G= /kz :T^ :ȰBF{E9U qXIZYH,tCyd q"a(d]K26](k +(Kyǐ3 zy:ry5..H)31qeRAҙXveRtzi)aW5f؈az$߳<{9%`߳1YA! LZlƂL\B秾Td91LWd~S]  oS%Y!d ׭/ԋG9 endstream endobj 385 0 obj <>/Font<>>>/PTEX.InfoDict 388 0 R/BBox[0 0 360 302]/Length 3272>>stream xYɎW*x9@q|0$.28H?ȚHJUowGNvd|8^|w/Kގ_.%rEHexMP+-3zn͋חw7w<w.i;JMnQ9Ҏ.穻9ͽ{ˏZ'VcxX|_y}«Zz9'+hWwsۋǽ.WO\ܖZo^ ^]s'RŪY=ߜ?P<`!.H JZ_K>Koq[h F?0Rkt2w?o_k|&XSI_L:߿ 9'+q/-3nx H}LtOq.;~j>,}işۮޜV~|`> _ŠUk}՛ͥd+/8 ]wU'0W.ז|)U 4_\y񜾠N|eSHH#[]Y#1$矩 4|l6!g̾zs1{cPd~6ѤE%5r54gj !)T`PBk^twC KɥSԳˬfIGqj,а3Vc`F.|YT!eh04ճId?| uv ʝQ4bΚZE63N@GR[GA^fRX ,Jzc:ҭ'N1-7bVsnՉA'ZbȪp`Jz6P.5L[WT~4XN Xy>J%L1Pc1113c ӑggBFe\M9v3WԜsjNG93qf <89$1f6Dɇx aׁ$@i| ~wG+Ά_l,M0:ԡ Nw в D0ɨbOdiAelwQʇ4oG'ӶhΡ1n_Amr~8c F7Ry 0Rz@&# d6;)GW :o9Hz'4<{!*_OLэ)*&C":OݐC*V\һ C]irtҗ$9jҬzN ژPG" \fTs 9>@"SF .m\x-}@P=zdDim ;IE\fm r5dd%TyY<^`l%kE=O; jRi e2}ds&׌ƣ{"_;d Qc;4R!w~BmN#E9 YcU07 !Fe 6X8,sڨ2fjhss4qjԡ*cba52 ˑ|4|>BVd BSp􁖅RiB15\:mȅKpYyG\riŠ(xJMa+ D<4]M.&Cճ!cy .,4R|4V%; e^裪^y8s }79EŴO_1H *'ېVNoe!CyZ+X]8V_Nsk-d+GbzHs<EE endstream endobj 389 0 obj <>/Font<>>>/PTEX.InfoDict 392 0 R/BBox[0 0 360 302]/Length 3153>>stream xYKb60j8){ 8A,VY קjW؅_ws"l?C~tV6ۿO7 Zo)o7~Vbrۇ[*C"=KI%鳻39?>ğ?l&y{ϕMcXdK=TtpJq_|yݵomYz޾}eJQSf2ͨecT+(ͨeQvhl x>>8ؗRg-(' |4?l&(IOas8v ܭ)+rY]$=?trǿ}-`砯E8q˸XVL#}"qE.E|\_@q̀; qd`ſ8Yx^53>\ eȦ`EaŸ#F<ˇ{4q仮>z8_#/'*Z`}^nV`EaE""8XnXnBNz)'}(qG/|k><%)F +wIbIܿfO?~ͷpTԶj߾է7mͩZ$; B(Yܿͷ%pV$$t|lup%om 1LP1TZ{Yypa^Z_e&'Ju#FQ]nI{&AX vUni$K5d fa>hD]I^`|3׍ tVBH"ab4d7YKrO뚊q_򀚍sIa`R7$AH("9 [T@hEfE6Y &ŧ3xwI.&*ЏO[O=|@~QƼG)e;wbq0 ́XJDTVQFԨ y8%d{L*h ,U,D-K t9Ds VGKûz@$h!":;|:A4[[|FFn.oox`ș[#w$}Pl6&+7‚d6=mejZ6;VnJ:u-:@ K̵'oDBTlX!越օtPVqv@iYV!(VQ`k-%9O`+܎ZZ+w%|( df'kVf{i4M s t MDb A^C7j8n]f'{(mOg[N惬qR@'Bf$'ê΃_ܰjԟm@2 l#9'morv(m,iB#5 !5HNV>YUT$'~;`K#\gޙ4zcM~Fem] eW{vj4WK5_!kɤ PRgגMd] UlF#&c#aXWX3g%\7TWd.ΙBpbLb-+)R\LnQw`nV5[솕b̊X솺C7j}Q- *QwNP_hd>솺sr($2†Evc 7tޘ % )5Ɂ^ T:2K'u8g ))$>Z>s*Fn&}Д-*c_*:R1Ʉ+E#/q jHeȨٺ+AlQ܆lKmQn6;Hw@^]d7m$/ Lž,ݘ4#]7.f෎vTσ~%v1f8Qitø]{%wߨ4TQ Fuc#Wyo~ Ql< ܎mE xIq>f %EJZŏ'V+4+%BDD''*T.٧>~gRJ,ƣh<%F%Rt|T&f pMŏ@PlGOs&!tժ?h&!"mÎ JEeIcَd^VbĮ6](-cDuRqٟ?y҈S)%*Y4O"ۑ!jl?KFm/x+r=`2u [T\£:}- (c:M\ {tGFL:u3}H,Ic_抦Šd }+V~X)M\u3I1)Ӑŀɢ=Rfȯl@@F2W)m vtGJLq/+q1 '@@J,ݥRuQ5ښ yح*EW"T@:2WOCbÜ&*WD[~a)Ӕ )D]V{ΓXyR$k%J#,D˰UwڪO:ąyh:~ #w]hʻmn A#jh.Og`i^^M="?mb44s*'^L4~Aūo{ endstream endobj 393 0 obj <>/Font<>>>/PTEX.InfoDict 396 0 R/BBox[0 0 360 302]/Length 3251>>stream xY[ }~탃"נE}HvM8v68@~!)5-aGCR҈_gvQ[pߒ?o|[ȹoo onwT#Z*%WLgӲ/{!/opD/HݵTq,/ظhcڏv1}-qpwN_zOmlڴ={בYLLXվ[hZhZV#JGf5Ef5ebU.Ve*WY[hZhZVǸ A$|OC3%Ei ݍ].γ`nLT")ӂii!L L E!ke.]R,vL+LEفneU.P_Rb)eieiR1vL+L|{ą=>?=U2iqbo?{2@6t^O<T0OA%b1I2**E2XK+pg$m0W?u`X;8#?V?l}}eֶW9r"Z! N@+]\]c[7|o`5߰ޯ/5!-n5_|zɮŏ ?F/9KVl!$̐m޿~v\ |Hè/oaVd>:ಊpIl߼WpRv)$4IKwһa(՛zI~T>m$u WIHfcާ1&{Տ9L)۷\%x羒/jb[$ k1?એ_ǻ'>GolT;ǯm=34ӌkpY$3b0KP)ENdwMJ؀\>y*e !WPo>.K9TCufwB6*Wr2 tСdIĥ&$#3UIђq>͈.({ ,ۉ92Yr ?} ݚb7]hC;>aA6`W(4^qE4dc\$1c䚸ӻ8  z䞃*puK(ӓR:<58Pu4{h i{"d >`g3ЃoCT=zrr g7/n`<^+GOΓ#MqԐxoL@0X^"`i ̖ %[.9/" }>*Sx D"=3`xIM^(f C5pe,W #x2?h !i k܅3\^Š݀5pC B>*‹,SA*( V恀A NiBap@P3[.:`@ؖx++,*KAF!Ene]Ŋ, "KEp'\,n|WET$8&Lݨqq'!RT%m7z8-ȭeF$£4 AC$6[VpW$R /1B$ bġu D(ANZ34qHgX"d)gBIZTѫ2Dŋ#(mv@=" Q:p,I5 C6MT4 RR$4г!+AC)bg,g.K&J3]KdUK"*@ [ȌJeh`5"Jp n:te1<j 'd@ZR%[(lrR-CHtJ{x2mT3ah>E:(i53/MTXFI\{K.+BYFe+#!# < ԕYHQbI3Jl?;pxCR̪?C MKfT V9 ʓ(5I%Z:!:X6@>*NKkArL)q8m7-Li5QgUY箿M+q Ml%Ъ=/I%yRÑ()oIϖr6!4\٧cשTWrCeE|LReR<9d$ N$!QNG|1>Ijt-a$ dc@ePNG8HchZ %dp, {@XFX |g"32qgk@T d`@זUA͔ѽ6 OnˍK?)+3/g`=Pߊ"ov-_{^l 7!|]_?1y\vvذ],(-\bULFX^7Lg{.b81٬K:#5v?_$x_ ,v%'Uw_o=g?lnB<ꦏew/oyo.bUY7Glvn =)Sx!uy=_% endstream endobj 397 0 obj <>/XObject<>>> endobj 279 0 obj <>/XObject<>>>/Parent 275 0 R/MediaBox[0 0 612 792]>> endobj 398 0 obj <>stream xڥ[[~_TMsKvwvכ؉\l ד_s$8F}\s(7݋@^|66ű<]{Yax97 uo*nKa xIa}uc׏\=с,^yP-֢x1#S~*}6gD c ` ; ̓U44p}?sSKjvFjn"U1\ք-*S 9 ۆ9+h1S3A*%=@; Ww}>Vt-J;y&?(HcS?q= +$W$Ӏ\)/+OӏL}t̑ _Ti./$ :rUEw@j ?s?jeC5I`cO&S|TC{;DŽEJnQ:ԬBL5㧙C `"#fnOhK* |%-աC! SN#;nB妺{uDVH}S]S2(՗ ja`ffE0#~!hqGdNyxl&16zzS^if4`V.4|]پ̹MzOC.Gw?)(؋pww45ێJ)~ҋIyC* @h';{ L371p^-(m 6S~ C~LFPF@n<>\oxuϲj>,8lēI؏d4:,vǺob/q1KzPu T<'T:yMIKY0 h`3A`)fC,r0:cI S#޽37FJZ,̦7 %9|inpo%jMз@ЗÚr8T {<X:y }[hvDJ2ZX݇ / )H2e^qa VX'cLn+賏Sӱ&S %9ӷlaI"@ 9 *ˍg *[*<vR1~Gؕ%vbfO"DŽ]}WVpbD']F^=L] əjn4ț|W7|bMu&Fl`}5R{qVgA[v Nt9- ud}(  irJ`EI%8:Y;qyf:YYd$"Dtj rlQJy*0J)% epUNy9\M8qXt ?2whKMI.<ul~J"; +YZvM riHom [/aU/lnSt"w;Xð ?s doT8,7i#j%Q<붬 q **(VQ'-~Ш}`r:(F>pP/|`9/@A#y҄evC-/LČX*?u [/(2aLkeG|4HI@?q৻:gB!R㶋4#zI&VI\/3DvC"Kn 4l`CbJZb,CahGͩۀx|aO k` >s>1@xIu`é*c%a8![C4h #ptR]|W`YԟML@- e25?IY/†ʓ{YR ;ͻ04d%Ñ9VھX&q(B}86~dg`E95,|_ b RJu+1!Qb^!WVMcV @ת5 ΄{SE,ɦTc{>ȓ(#G|xw;/Ha{r}l#c#n]Z5CK6 !نNœ&Z~X|rV~? ;[~ q{m^FeՋ= 5;`lFkg/{N}AO<W\fY<;Sr +]{  Mo9,>t Q2"!:B>>xP~%[AcM7X5ăGM ֫IYsE9'+-_5Dc˅9 ˏR^!B$.]4ͯ++}k|sn*s%M p8J.v[qjHɬEriXl r4H(QsŤk7 ӊU|6&L,l ,7^@AaNHH +Dw0{5/]tQ7>b@E g~uWr{1/44PN C!!m;fm2vܠӷ2褡 hln-*KS#lj qy;% UaCl.'7摟Qi*O4΁‡O߯ㅒٰ.OvA&J O.`bv6d8GxO;.ձ[_̢fE_b",oxg2NrJ ,RK؇ cW M0"RJqoocvr>y?qkt чs=,}k+}_t(\fcܪ#L ۈkcmqW;<.P T[/'sh7(Ve _s?_W Zz8@+h(*CtxT|Ia //!ErYeS %EcA4Vļ`cR6Mv db1ǻWkg4R9n*hcI7,V{PJ P}/&ȼ[ѐDтV؋%ږ'O+#4O`;bl[k850^*gxr .A!nkIgC7oy3L)?@xT NQAQ:bhi{,mr8/umd{y篿])9‰ @uI@1|8:MY#m*ε^!ro ߛ3YYp8̐l3Pﰸ5DH틬DnX\9qOJ4ã܆!(%pȿ]d.@a;.]1-m_QRmnq|- p[ΆZzaƷ2zJn WIXwyMOUėhZl3}KA kDrP3gD8O#뻺+[l0oJpq}avP1:׊ɐEh1® (Q-\h:=TP8⬩Jݱ$ N (ӻԎrMej]-ob(rdRs',_=}f U*HlveRk0fC)䨢\!O  mr,[U4`Ei僀~Lt{A qM1rtGZ-x] RLGwD**ĮҰ&qV&<4MYG^K[l>_g;gBkXܯߙ.vf'@Ax~cC>ۢ}_OK(Ԝ/dWBIRp`@!r͔<fOI (1LIt_):혒%$q% څLb~/O}ڧ0uaͲ`3 v`8VL_z2} ͝rcb<5#NQ ,jκ=clNH$;RNFڤiUD=ILpIPK1;N6\ڈb k!X_%krRg%KfcU!9N>.:D pqw~8Ook74JYA;Nf;Lef"B/,t%,zEGsor!#ފDBTOs9*::mؐ[I0N!vd=;`b+nΠA{4u Z RwLH{t itʲ90H@5bA^,#ĭtΣ%(r,fȧDgͨP_/+/}FS~ l{L3TWա.KR5.8YM0mQt{LՀz-߾}Ƃ4L ϧzZGPaSS\#3sY)Ja9օ+\yOYVrƥF !t;yr8rT<zl|Dt}j b endstream endobj 384 0 obj <> endobj 382 0 obj <> endobj 383 0 obj <> endobj 399 0 obj <> endobj 400 0 obj <>stream xU{Tg1CDShXEZS DDA-(]AU FZ@0g,zzl|t'Ԟg{{{?G$iT+)6ɏw%i"5C"j:uD^@ rƚ *'AdŽ$+cfn*Lк-tXn-Lq!|+U[J6:S1/mu&-VJ} pՄ&,զ|\LόSfp ƒ$DOXNDrbED {p" 1$i^aD'9$wq E rK;av(wJAZ /]glw;}s[6]ė{43oiN_y7fH~sDvqEF0B.7O#]?S;4bA΂oWҮ&~XF!v/PpJRQj: P+AP/8Fؖ4(1)Ph٤5FYds x>cV{ lC>֪e X^YLQYQDe ? ,!fpӵUsQENx2$o9ku ?ʭ@*g[JWf~Y=BC$P v QR|6u{!Oq4 ƞRGx!Pۣs~("Ù.9Av,uG7ĊW0um#i2ch-y6ӎ>mG!/?nBK#`Zj(]R6MQ$~Ny:_jw՞F6`:b׆-K>T)m-4qG>` y,3LՍotW.?\+eoǁM/l~!!Cp__jvߋ V2fY;E祥KrJw_/[<e(E{7oV%#SOGMɑ Nt/= Ж9Lm+;jM'/O|^Q{[g בO~`pUZ)G> ڡ4JQ<5G(ƾ10I+ Pic3R4Ju S \p25Jے=#JΡg7eXrcьX? @.˓#ܖ .x[玎޹?;+fKX Nc A[0 fR)d)0Ԅ(MofwnZf1(l|MjCZDӅUeԳ c^}+a24u$303ucbp2Uyƹg`0cb~Z> endobj 386 0 obj <> endobj 387 0 obj <> endobj 401 0 obj <> endobj 402 0 obj <>stream xU{T!β&9b5/@EPaAE5`U|"G|j >`vau# 6j=wrx n9sfͽw}{?{$ITkyZSfZ)y_O!ltt DKx'aٻ;H/u9>h`w!-Z+͍22%R"AF✉lPf"wz۞FGO7~mnAAi3jY)rq-;+I_F!7|"P3-z繂 ( 6gw9eHښ&%%< a#y΄3hA-6Ĺ'~C#LC4P |b9L;r|IOp, x.S`m/#}9T7O+yg}к eC;A3ܵA;D{ b^as78h vMc@kx &%T-YҐc rk<~lbɶubwLQOab㾻kR}i<67^ܫ5Ɨ$PczX^I.Pk{dU.{ߌX:9Blc"b3c-'6P]\XlnNݰo%Gj+bicZ2]a+6v6V@sW 3*!Ys*knljGwQSUS'ŠkeVN(9K^;w᛫':fy b8{k8^ nw.5[`[ξB#߇!ӽiϛGKH8.G/%6W;3LZD(4ռD^yN'-Wj~^p۵Kܐ ,IzV`DBɛDŜf3l8Dc ³5N_,D0Ap3D,z&r  YA3|xz5F88 c KH5`pE4­ H$xlC]UDn Zj\c DBNxSAj`/YߒJPPS ^SROX+ó"8ʗrAg⧌Exf6fhob8͋u_}϶ӞZZ Bj}zlvۚ}A꣕ߝoE—YN>s%V]cypaVQ7//% ښA+fNKf\*qPx) qks?^j~0$}}_x(W6Hh,;{䌹۳툵6O,4olW=$<x a= p8lCIO|3,@ 1\WQ؋UT8sQ* h=0zMkzB׃1#,YfA*nދ(HLi3Ę+i_Y?9 w)V endstream endobj 392 0 obj <> endobj 390 0 obj <> endobj 391 0 obj <> endobj 403 0 obj <> endobj 404 0 obj <>stream xTkPY6IuU2-*( 1<Gxh"@GƷtAG3tDQPTFd6NP Z{ܾ}IG$I9ruzRpZ(ֆpqqw~,+6#ˬGo~'IOYD}Dh GGESԶnsl7~cX}JJL\mʶ S$dN'/fqVv:!i<=,\:c[OBx_aD8aOD~bb@b6Lx!hb aF',K$&bvM=Նn\G*Z>l2jY~!b̏Zʀ1Ra=~jJ6+fWm6I+CPQ%1gf=Jt‹uL2SכdBVS>mzm` A 1BK0dŰͰ@S;<O+akvj,:}{,ExgKOS@ E[E5+|hѲA9[:uu}]?"yA^WaR[gsYJoDxˑGvV~"UT5; (xR=W%,FK3_ۯ%qZ܍:X]:,+?-&3ַ+IEA;դ]8m&x؃VsbS(D@Xv56wF9=1 $6O6/>deF5dk8S{)GZOF+MCx5;$N!%ivΧ9X.v^(zֽr / p0Bβv@D\Q h [c ,Cxn=g[;Ly<kiπӘ`ydf!p té~7GH&(*4Ap6dv_. v.^^q|'n) `[G'mQ&J!F?tux.q䴼m@>?%\N7 f\ ڬF%X}8ˀEQrI,jFɇi<fz9-}k{} Ͼ`E;}$LF@kHeq k(ݘ޿73l,A> endstream endobj 396 0 obj <> endobj 394 0 obj <> endobj 395 0 obj <> endobj 405 0 obj <> endobj 406 0 obj <>stream xTkTSW$^d &)R"UPRP £&ZZhuDFA +(VX@-v kLuYg>η7IG$I Tr4Eˆّ(nT~ShXNbA ؏)N+8gX?<)2lT׏flLKI7-Vf˦jmZYNΒ'g”)٪?N47fqf&(AL PEt5{ldGXJL&DI8bŸDI,!Fc Š&{KIL D#3H&-QΏ`@# g #5g"_HGI#.flM'B%w iTukO/mN擽PK~IK"X~)7VehK !ۺ]%F[.+'jj.hЀ*Fݫ8wc"j̽+?`eXȃ@**yz#j@uw.@.-~ݭ%us$g0Y)/n:eE^n["KS۠F(xj(H r$8DQa;k48>tKpg('-NεNY^c,;{&8԰KR$Yԟ/gzC?^(7 wk4J ځmسf˒uU(qmY[˵&3Ftb`35iM1++]*,>=|@Y1.+x"KvϕgsSt<nX} +zR=W',F̌[/&JG@=$B¢G.դ=wu*k4,Ap=P'\ؔ@J^f72zPӝ^5MŒɋk޹|gOP>ZK^V#G;Өn˷Uq>x_2Ҙ]2 rUB<u"6 ,/W4*]M]t 8}?э(]Sn0wuswKdw&#FncaG3Ro*W(1Ekɶʃc93v/-s10,:1S`{f"p`DPxGx1Ǹ۴VWZ+@X S nPffU!Wc;akpnfd%GIrChY b]q8^j鱿BD?.}3I8< \k.0E/U`z$5(ԗ7+h+m'$T8f@UҒL?f.)tb [su6S)wZsA9[DU狆GDpjQaTsjʳ~80\*kD ^_w蔶buDoP'mfmS ǼHx|X3>rwDiL 2 T7~?ye=ӝ;v1ϛn^Yp%'|E a*~ F,XâDͨ.HlZRG^!ngH++!`]=(}b+l=:5Ƃ8GIe'b>~ɫF" b-\yn ;'4mzzn<^' ևJ+D؊Tʈ16J}E xI)q!U)"Gcށ_o{&ފ=%TFM~S(4R˶tAVSWخ|j'> og^эjoVӣ. auG|1LmcdP3?\nںB ?eT .K}nRSB62 /-Ŀ; endstream endobj 407 0 obj <>/Font<>>>/PTEX.InfoDict 410 0 R/BBox[0 0 360 302]/Length 2784>>stream xYn߯~?F,/$ ;@LbI sNUL_$!@9u}?mޅwmǭlo˟OŲ~Jo?D{Sy˹oTFǏX-+SgWe9_NNAw[Bn?ta Vk&onU]l\v[n>6QlpܢRieѪJGDdj6|zg sa9UEW]}Glv/:X ܙ&&(h|N]8 wˠ_.>uXB?BΠ!C۝ eee`V*0Ea]KfeB3+Чv,4W/Ĉk\?G>34th3aR,5v F%bQc\ҡYL dr 871^_'o MhȘT,{fQXֵthV&4;Q=S^01^Å˼p^+2s87 '2T,CMfQXĘ׵thV&4;K:0GyaWT/욫ek]2/;1|mNJ"łPYD,J,+:4+I_AuXRvH%S9f9\<\775/q`S!hu,T`* Τg7ʼniv `_~\<@/þtK; \Z)ttcޟ~6̏j̾aw "wHNݐenﶟ|;!yE+PEݗ~m𿻅E(2a>|wpR&'$_Q?SCY:R5:^ziLRz S}y+ ]?i4" s 6CٴԦ-U[璛 k~#n+p?,w?s68G{$ٻW䰓Od $d7Z#0 u; K.6o_ͩp-`hx;5O=-5 P =w6hAKmrFcNrb^a+D|l;3x_j_l#E'%L& ?,x7)Y[Ujyx 1O6j# ᄀ"bNXє)p&"0tâ2YqS?mSQS@pY6" "XFbdc|w uz# ټϢ3x]uҰ. t Ō5"1ֽИ 8Y诺9R:>@Xv<\ "z:yJE:O A$: NCi06(|LBľ \3CߘB!M6d!M'tpfI;${PD:( O죨 !R!ك7\|Aѻ*"4T##GlRK<nD*ni`>y!fk:;zv9irw*HC3t)Jӈb??Z^DZqqFN53 l693Iz=}4Ue8J M񏀎rr5g{tca<  n> qQ㵜|}Aiiq2̼b8^+)ժh[;5]=*>/Font<>>>/PTEX.InfoDict 414 0 R/BBox[0 0 360 302]/Length 2546>>stream xYɎ]߯Kqk$"BF"V*~hї8:5zV˗5Ǐ?-żkd֔HA??.w4#Z*YCgSewcϿ,`{쭭!?/ݰRm?-o6.mO˛o?/ou{9l ܈HrJk+ )VSF+`ea5Ej^tD/!o*lVp-xΦh0e ^l &T-?YpZNᴜ e"/A? [lo|u* N8T- N)U˔N6S8-7Epi .E9Mp:r#6‰Ɣe~`Z`ژ,?ej7!}8^D*n>^b Ǻܜ &&&E%m:C0,D-c8pMec(uG^BS47݆ͣa[яr"c ;.͙`bbHL L E=ZԃE=Y:9XSb|1'uSO>ndfu>9wBW93$jQs-8&&QL L ⓖuTOͣݎ>ͨu B}V:67F1ݴ8Uڟ{}ε9׶эk8d,ncqlэG6ܵ~??KķʔEA.~/?~v04"nN(˿?~yÁ*YtD s'़$IG(#zywKI,6 4r6Ó? Lrhfr XmZ7:(1k\]? bP-~ا3$$m>Qwȡ;-1N.=Xm@$`&]Ҧ· -UUEyK/u\~yJwZZOXЫW'E,&]M8N->cP~4y'xc\૎̈8+ DV|:&\C(դۉ r& =xكwKp>{Ix(UB?{e:sa@j&eh+u:kV"\M:-УRᾎڼhtB{Wo*)g)H _MkJaXy  qpoPV EK/T$G)`ؕHB׌mK+reWq}A:Ah׬J#!U;|M^GB 4Oh&CIb'Bȱ:$v{jQ*#o5@w'MJŒIf隔G)LJfA#I ó^R3ڙm3X+G(̙#s4)[-f⃓W@$O:OedST yWUʬg Y/42k0Cd`jLʾ ^9!VP .TM ZQvI Lf8>y?NN'E]trl)b^E@#,F;R-z&mT$c@r&>h_8Qz-6^Rء?hwj<Fz3D9V"HxMkGa@jP#\+GJDܠIT .ZbR")hjz $HxEF5ٝ2|_>ct9Ēm\ڤkn}66 SȨ@$ ] >Ay3r@u}J\D2m}syR$HkU-TR.+z,Fr!}֭gP@,Ų>w"iIMDhѲ=f XGnZТ6<v"zXF:vg (mDRN=hE2oHmU)ʎKy]TAQς:@.z&ܧȷB37zU/*=JIѢ*82֐ֺW?}{48CVFzeBsaAgB0v_pÀ'8\֒ޯw˲;v~\Ev:IlIѧ̦n׮dN" &_fWh zC|qR#\kw8A9(@FI@;(jVgogW endstream endobj 415 0 obj <>/Font<>>>/PTEX.InfoDict 418 0 R/BBox[0 0 360 302]/Length 2799>>stream xYM ϯ >4nF rw}0dɫ@+@T3¾{xdw*װ7~_/|W_?/?y,i^_.&~%%.6|@ӕq䘷O3ssOnckml-Pq}qd_5M-djӈ,N8$2oǬ?CM_ky!X^/-~̋´FMl?N8it: SW`1{Sp}J\m7iϲ~x嘎 .iC SWL#Y>Zsqէ&(hJ0fΕ=!ٵ]Zh->'a%N|aZWLJ8[֧Sk.Nj-dj/| bvg>W[-=y~ed9|oNШtղ^ #8&cvC8a]yOpZ`~6+O\woQ̊φd+O&0{8ա14W dOo1,_j[g|#^FD?rڪ>& ßnp×Y]$Gݶ}wfmg#^Zo?x zٮt+n,P4޾{/f᦬Dcfi'O,53H\SF}{zTpb.Vԏ:M鹆VG:4Dv5siSwi%Ѿ‡qG_뇅~ 0bcVl6D_{ĪMzJH+,#;HKUv#eQFϓ+)=J]Ie#kb[(6߅>C  -h%5sA?j"@ЙTL(ĩ%ɮnW0Yܴo.c4c㓋'"K8`h"ΒbyI..Si$ [FoglQWx`No-VDn5.ӵ[dB= ]V¨[цM  ׺vTA ACQހlY[:P(5R*:y#7QPFRidG4 WFr!vm MB zKe)a$f2q`z-`T#`lD3*JiMK XePЊ&/VXzXE'+;)Іs Xڍ z +fR^. .SiM-4\ɐkuZJZ$cxd1dI6Ht^Y#9'*]9Ar_RƉ]Rc|g"ߔcqpt!R 8Ʋ[tt x-&;Y6elDmIVnԌd d=!8wPcYFa҉73d,j1(RC j$KE82 d b lgxC^ '&Au2d0?Lhd!x*aQ o0 < 7M) C`}m"&͠[1D=x+:eҠ[ޏnqS )$N`nqՈ0[AVrdGW8}@ .g"sr7[һçU Udkb"2aBʍyX[jq$*Q݇Kbp-/)y+ɂKHBo1򒂄p;>5[I A{ˤ}DbBl*y[BAj@l؀N5W!B=[MU_|kIGb7V}um2"ԋ%oo$UX%0I!zav0l<1)#diy߾FZ߅OH6e3 #Q4:)u\\S4r7Qv/X)%`3+OeC(v !u23"[Y/Aڐ2#flhbqVڬ@-L|,>;rRkuG\ihf6jZ?KmV>2~^׊FyLJ @w7XnΨHҀ 0/12+a㞢ӣ%m2.ktŽ~/hAUeYlkhIg*j8/`2nW@+ ʞb*: 3V|@Fmd:B}yIndBQ{sd8=' endstream endobj 419 0 obj <>/Font<>>>/PTEX.InfoDict 422 0 R/BBox[0 0 360 302]/Length 3511>>stream xZ]}b y tw@]#˖`࿟sNuϝk m;=]]SN[m o}pmlol]޷^$}K y˹oT+}W\EJ*{28Wqyy}qvŏk[ { ŶJ.Y?.K{׾gNWF錧zGN#+}𘱗{ɝ&x{{4[ۍ-p9f2z]ZkhE7{pZkݚ}͖J]W??!O||)'q>-|Vɵnjeϧ8X|bXx 9zKp}I]ml[$ǐζycrukO5>)õjMuki/2Ĝ`DeX?r׊+$&ӧeD s`>,3Z,-õjMukV+{8Zk|kFѣV_Kjg[=ZFYsR|kVzbHq=sgKi/|aÎ_sO0 ׸]c9zα=s.Ƨǹ׀3wNnjp)tWD_^?ǰ{_>q7c)?^~Jc7VPR޿}/_\BdɒEI2M“.w,mݦQ8fńK0c&^ւ\c:/-kN3|N]W֊9)ŵ<.)M);L?#Wg>pOn~oANb>k%Z@gXh#`cqϪ2G-\2cpgԁ33[ti=A1Z0qlȱ:G+Q%S>T$cB'3I @͡%2^ {Q`&z]ZMdAX}b٫å[=3N|t9x h8wXFD#ZfLD| z D6%):x5 ʕ;DLo%`-[:xP^(&~mugwk3DcԃbfsfR9s@8cׄi95r "sH)hqzysGM[WM Ҁ :LJҰFFJ=+}q%i b,^aa3"˒dFe<]lvllYH[dp.HϐEE2I|Bhg PLơfF3]!1am65x?ڈBd:䅈e];Coym@I@I`:f^Mˠ6HmtiR:PL!L҂B- e(-=xKR5VJWƀ1#Kv=,-#JGo Jq5,Iʃ".k6$ DQ_$mGPA]T'bs?b+諾ⱙ3Ry'N1Rf2Foy>Ȱ{z|>~ io/g!u'`ۑDz(ue F6Id;bDeUQIeoE6x%nd=J-(#l ǪJ| r]ňj?\TmkHz$V$xy(8 +dQd@&LKM1!dP ǰ:Y̋ 4 KU}u\yEz(pMDYA'5d@1>%5ONE_aIuD7".YQЍU<11<ÒA2)#C.2,*en-EHMs#/ ">>8V ΍Ġf6^Bh,TS^fS3SSё<:s<)m77VћutmUge[7_9QiYWcl_z9#op(։ޓJ*%wPAe^ *QL`":P@62 "Z1oat5M\,=$,h=eЇ*Bz5H$vц}`YcDg籓M}HғX'ر- X C$ #tyK Beœ?5$ ul"]x$V X ꢎ$Qd-Cq-Z~7\mI Is#sSCvad$Gn]B H:ޗo,t#;ϮFEނ:z(_Bԗ"۠^6$Z:s\D]X-8e]q I|9%@ p- MϛZXWfj@yX ;Ԅ L Ѵz['ȂPxC6WlH_jau͔eWWOIY˩P +!IES\%OIA(_c%S|Ub鏦Xn*ɗSe[8`9t&3W^!P!NbeR~uĬnfC?nA+/65WNâN,俿!}z]ٹ:.3^ƼLfy]=o%4fu|~9c u{V^b=pqw~;O|92NJ> )UrzJ endstream endobj 423 0 obj <>/XObject<>>> endobj 280 0 obj <>/XObject<>>>/Parent 275 0 R/MediaBox[0 0 612 792]>> endobj 424 0 obj <>stream xڥ[[w۶~ϯZgu&^{[nڱcy{7ERkTH* Mͅ5Lwo,ٕNz3y\O|>mYŸ<&f9iVٌ3۝ *w45Z[VQUE/5-,׊<ş)M^eش{p:8:r^84kZEEl AOW(F\5)ǙpY^{K7?Ffi̜]S(ZH(ָ"Ћ"FiEp䁐w4|V&J )skA$3Y@>0x-\懴 ;ɧ9_8c3q\v`A )LH?l۞U3Y]Hq(" ׇtƤ݀Y0R^.ngYӟ_<, Zf%|כIՃf 0:Gxt^!oj MzRN}/ [j$Ji\eM3 ΊQe4ߥ7X?R_G"f'}08?a<P. 唾Mήwܞ\ڹsԐ̂{b\(;= |1}O2)űiyoSp"\8ug_Q:;ryB`Op<ǠDUp  .`LvG,Pd&er^V;zIQVUǃѩ #&)0ڐPDUJFnFICpJ3tqVVR-Ex̢AB-$uA0$  i)8IuVE p}*1(~<#4b ^4:%xVKӮ,# 1KRWnbxhWfB(b:]8YU>CԸ)h<ƻCBU2AuU冃Vr W=&T[Z+X<=1B"B]S-y*7]đJs3 ra4MQ`SE Ua]2?lS O.-Th0ʮ $6EAէC HmA t.-4*%@ ]JT$s7ī~6J|^Ƞd]E]T)3Te.n[< , qY$, Z~@0"Kt6(|H* \gbZ+97ނhḏo=h Uw^G;5RIY[TV&+$0R>C&+ .!GEOXEӦ*EBQöY\6SQi4v-qTnѼz]Ej de}bZ9DK@i1HzXqq<َ}/#L89l`Qxp px٣f5AH,PUIú82 V:w͞n2(obH6ulݐU٨ pz?v"tƌdZUBg[DшMM6a3 '<LETMJKo(?s.@)DnLiئN}!XMs*Vp$6H xZpF5Ѐ7zE5ͤ4:&z;c|j7Nɍ;੩T81p^UȎ^#e*ijfiMyV/lkef^?Ѕrc&F\Ƅ_ Zp{Ds̒㢟r(qj$ZAfp* 2%%V4W^w/^@ltݬ*`i̊ T%Q\ ac{IpLd攞ro9 G}۫*4mP9V{NRw vRhM?22B R2(!t_W(Ϣѫe~a+c{9(9Wֽp%BnJ5ԛE"2<6jz TN tz񂅶:aaqxXX, }ȗa | PG sρ{bGK}:,Gc4oΞHjsֆFAwS2)Br#*!׀`MV_7tIFROz{=Yp]Cɒ"PKuCzd29ѼmwQ%Ku2ѳI 8Xl(]AQܔr{>4}'DhQ3 N鈃s_ u$:FoL'?:}fYTf5KÙ%MCraa1(UNUITYd3-Fu*)22S{\ݢۿ@45m5{aYRggDL#Q6q\K/X$~0+_(CFWY O1x͢"g`NaV3QHtkٖF`RrywS'`52pD7 sJߙ8,#7cЃ'_Mz?}Pzopx4)Bńr5.+2hSWAi^(`:* b}+^s<]d6ڜ2n#s{(6Upc("0Ot-`~x=䳶9yjX}ac&b|Q弚R:e$CUcvjnU,5lWeը&J S nOzF3zQ ',ͦ2:R. Ua]tMsEL4Fn,/)ԃ>dMoߎ{53j4K9ǶxASӶ0$A=9/Ho| /;G0}2 c1fu; #i7Jy௥zE_iTSs_S!UC[= 7u3(U]’ZUGUPXeQ=@/x&M=ܳ0e[caY8mneU@I"RM|/Os!r0 0}V= X |Xo ͭ}Yw T~0E]{S{6ke!BgJu[)4}*%?h1EnK.F*u! 8!#>l}k ZIdv9ӚNr]ҽ4U C[=-5#-9@(/ћPc@3J+,d~%Ī֛e^b!^ޤaUCK6ir\LA6ɑ:KtEgRMd&0#ng&(=_ q'ZZ n5r+䲯: օ GcSo?nxW~gYoB;si/6}mڃFZ9T_0-k ZDMZqp*X1aƢHLDlL>;Wp3F:4yBGmZlR}EgJ)7 ꔦoSP>=YJJ%sP{kG.ȮIW|H d]OVg=uZF _.mC3%_)Fgf yC2#l2 $%^LjL],MV.{(sx5XcWDͥc>}8 w1z^>s"-!)Ô4.Q.rJǙ~?O57Q9LN٢h?gN:WYC!Xq) :~c.C/K(Ϭ1H\6H:3Ci>$B4u/J+.2\˒yq,j2 #^:&GtE+\fe^y~;l~>vaw*P_ujY`&JG\̶(sE̕S-;08., endstream endobj 410 0 obj <> endobj 408 0 obj <> endobj 409 0 obj <> endobj 425 0 obj <> endobj 426 0 obj <>stream xUkPWfIE($*hP&Dt5F F"g $ 8 Ƭn͹Sw*{ܾwiJjG4=4$YIE^ؙcX "ݖuX*|aXaO!{ rG¡v%i_utktIsVgLKIVMX|䬴t%7Y:C[JLRE&h3g鏳)RIT>92-j6F:}i>(j5RT HyPs)Oj15ԔNRj(eOɨ%ާʑ)BJJS ]L߱ݕ/ KnK={f SO/[Vs#Bȏ*e 2l09tavpیp 4xrt3q_!X3y$nʂkO! lԭ\^HB 8!~\R=`==Yz7LV5C̲f<,y, ȃ\" Wj<,{0?>ˍWzn'f\&X˺$ڎ.V:vr"{oZZdxgv;hzaKL(-6&V$w9 FƵ7cV.M.=tx7 k "Fu76&\wR9XQhÕ3Mj+dP݊ݚ. 1~㧓s[UltMz_V(Z/vn.\Z%g"11N{f9_\G\6s ċnu|mGӆf>]W;}J$AزkڎR]\OEVFn"J#:Vá [ vFaC;ѧo@q>L`޺>3;M8īnT*zz|j}bUhEE -C"~8 dmg3;XmM}w@7*{-:ƕ%Kd%`&px@ RGZD+ $?PLϷ6; [x&)܋A O#^$O8Ѩ,z6&0Fd*"+ba\D}¿K9O%f*F00PO'~!GLD|#DLO0 @4 " b(D[$P 2o]hCm*,B("nIxNf9lE(V/AE"KL+@L݋⯠eA\,X,qZBm ca6B~ +X_>[{QyUǝ[kr4ME!ڤ }k{+_T;rqXr",d+vo#0 "e}Lq6h%+ nI})W*=O3<dn%d,S]sq'n!XtV:]ʬ|B~zp렡Sl@'$tejY#Ont$L?h&F5”^!jX%C:sE?+Y]bONW7PyBj6++ kЫdLr8==_ endstream endobj 414 0 obj <> endobj 412 0 obj <> endobj 413 0 obj <> endobj 427 0 obj <> endobj 428 0 obj <>stream xUkPWfEEgXt"\AT4D JLtTEψD\D@Y׬n͹Sw*{ܾ9wiJjE4=?AD؎$ "I$*XaM!k M=ҴG`xcx.K6mޚn?u/MHKNL*d&7m$$keهťه&$fRg(J45-=#3U䰕t縹Sdj>G*ZISB-fReHSRB S֔MPrj %P4GI\,v ;V1IE\%ff1O azl57nr P%3B(2@]X0m䥫5kOh@4GUMPs֍Q"X#y$@s2to4HT5[RSV  3xEE{*e3XoSx&k4c2jOLBɩLŎ{"2sxV{mi~ߦBuε>IYYAUzvjz$pX)LG,{lgkRZN(B}g6'*16QZ(Yi+3yu|_yfGν%_ .;k}>vWG]O%~DɈOTmX|_G+"J MlݷqGǻ<\MewϤ7haaL?mڽ-5=Y(&T5ݪ,Q<]QE oGWKw%x3`Œ&ꡞ!,a03sHcKUb6[a_ YJODJ|נ=8NS`{[Y+;a6~}ORfz@>< Ic!cIIԦ 4XP,rp,yhfaXEb!g{n\J~1qr "!ApV`KEˢQ)t=\޹FU8^ C u F#@jc{jU(Hlf<0elRyޙ:fAƫLQb%Yl/5!ә%Z=6QTQGoaޏHڴV-bGC1pGOomw1_A, nM|%ēW q+/wE[b'hTH-\;EPӈm Dd6"s7Ėź(XKK$Ix!ls p+2YG ⎈[r#b~4Kph/Q/vCT/ / %ZhȮ%z=}mj%PaQk]=#6NvPd-b} {I_չ0f͌3{J< H؀B\8`S:+LY 㫢v<%``~}|mÛqlnNڭKq>f#u|HjKCʮ8ӄg?. ۰:(Q9kqX뿛I V.  Gu}9+#f9GuR4mkOz-ԿÄJyót'OĎL^E+؂kEGOi/]/x}]ȭ/i}{M3[  H9Ac@RYHxZpAe~x'AQg endstream endobj 418 0 obj <> endobj 416 0 obj <> endobj 417 0 obj <> endobj 429 0 obj <> endobj 430 0 obj <>stream xTkTSW$^d &)R"UPRP £&ZZhuDFA +(VX@-v kLuYg>η7IG$I Tr4Eˆّ(nT~ShXNbA ؏)N+8gX?<)2lT׏flLKI7-Vf˦jmZYNΒ'g”)٪?N47fqf&(AL PEt5{ldGXJL&DI8bŸDI,!Fc Š&{KIL D#3H&-QΏ`@# g #5g"_HGI#.flM'B%w iTukO/mN擽PK~IK"X~)7VehK !ۺ]%F[.+'jj.hЀ*Fݫ8wc"j̽+?`eXȃ@**yz#j@uw.@.-~ݭ%us$g0Y)/n:eE^n["KS۠F(xj(H r$8DQa;k48>tKpg('-NεNY^c,;{&8԰KR$Yԟ/gzC?^(7 wk4J ځmسf˒uU(qmY[˵&3Ftb`35iM1++]*,>=|@Y1.+x"KvϕgsSt<nX} +zR=W',F̌[/&JG@=$B¢G.դ=wu*k4,Ap=P'\ؔ@J^f72zPӝ^5MŒɋk޹|gOP>ZK^V#G;Өn˷Uq>x_2Ҙ]2 rUB<u"6 ,/W4*]M]t 8}?э(]Sn0wuswKdw&#FncaG3Ro*W(1Ekɶʃc93v/-s10,:1S`{f"p`DPxGx1Ǹ۴VWZ+@X S nPffU!Wc;akpnfd%GIrChY b]q8^j鱿BD?.}3I8< \k.0E/U`z$5(ԗ7+h+m'$T8f@UҒL?f.)tb [su6S)wZsA9[DU狆GDpjQaTsjʳ~80\*kD ^_w蔶buDoP'mfmS ǼHx|X3>rwDiL 2 T7~?ye=ӝ;v1ϛn^Yp%'|E a*~ F,XâDͨ.HlZRG^!ngH++!`]=(}b+l=:5Ƃ8GIe'b>~ɫF" b-\yn ;'4mzzn<^' ևJ+D؊Tʈ16J}E xI)q!U)"Gcށ_o{&ފ=%TFM~S(4R˶tAVSWخ|j'> og^эjoVӣ. auG|1LmcdP3?\nںB ?eT .K}nRSB62 /-Ŀ; endstream endobj 422 0 obj <> endobj 420 0 obj <> endobj 421 0 obj <> endobj 431 0 obj <> endobj 432 0 obj <>stream xTkTSW$^d &)R"UPRP £&ZZhuDFA +(VX@-v kLuYg>η7IG$I Tr4Eˆّ(nT~ShXNbA ؏)N+8gX?<)2lT׏flLKI7-Vf˦jmZYNΒ'g”)٪?N47fqf&(AL PEt5{ldGXJL&DI8bŸDI,!Fc Š&{KIL D#3H&-QΏ`@# g #5g"_HGI#.flM'B%w iTukO/mN擽PK~IK"X~)7VehK !ۺ]%F[.+'jj.hЀ*Fݫ8wc"j̽+?`eXȃ@**yz#j@uw.@.-~ݭ%us$g0Y)/n:eE^n["KS۠F(xj(H r$8DQa;k48>tKpg('-NεNY^c,;{&8԰KR$Yԟ/gzC?^(7 wk4J ځmسf˒uU(qmY[˵&3Ftb`35iM1++]*,>=|@Y1.+x"KvϕgsSt<nX} +zR=W',F̌[/&JG@=$B¢G.դ=wu*k4,Ap=P'\ؔ@J^f72zPӝ^5MŒɋk޹|gOP>ZK^V#G;Өn˷Uq>x_2Ҙ]2 rUB<u"6 ,/W4*]M]t 8}?э(]Sn0wuswKdw&#FncaG3Ro*W(1Ekɶʃc93v/-s10,:1S`{f"p`DPxGx1Ǹ۴VWZ+@X S nPffU!Wc;akpnfd%GIrChY b]q8^j鱿BD?.}3I8< \k.0E/U`z$5(ԗ7+h+m'$T8f@UҒL?f.)tb [su6S)wZsA9[DU狆GDpjQaTsjʳ~80\*kD ^_w蔶buDoP'mfmS ǼHx|X3>rwDiL 2 T7~?ye=ӝ;v1ϛn^Yp%'|E a*~ F,XâDͨ.HlZRG^!ngH++!`]=(}b+l=:5Ƃ8GIe'b>~ɫF" b-\yn ;'4mzzn<^' ևJ+D؊Tʈ16J}E xI)q!U)"Gcށ_o{&ފ=%TFM~S(4R˶tAVSWخ|j'> og^эjoVӣ. auG|1LmcdP3?\nںB ?eT .K}nRSB62 /-Ŀ; endstream endobj 368 0 obj [656.6 743 617.3 588.7 685.2 726.8 287 486.1 715.3 560.2 898.1 726.8 759.3 657.4 759.3 665.9 571 702.2 706.8 686.7 972.2 686.7 686.7 628.1 298.6 513.9 298.6 513.9 285.5 285.5 493.8 530.9 456.8 530.9 456.8 314 513.9 530.9 245.4 273.9 502.3 245.4 816.3 530.9 513.9 530.9 530.9 351.1] endobj 271 0 obj [577.2 603.4 905.1 918.2 314.8 341.1 524.7 524.7 524.7 524.7 524.7 850.9 472.2 550.9 734.6 734.6 524.7 906.2 1011.1 787 262.3 314.8 527.8 839.5 786.1 839.5 787 314.8 419.8 419.8 524.7 787 314.8 367.3 314.8 524.7 524.7 524.7 524.7 524.7 524.7 524.7 524.7 524.7 524.7 524.7 314.8 314.8 314.8 787 524.7 524.7 787 763 722.5 734.6 775 696.3 670.1 794.1 763 395.7 538.9 789.2 643.8 920.4 763 787 696.3 787 748.8 577.2 734.6 763 763 1025.3 763 763 629.6 314.8 527.8 314.8 524.7 314.8 314.8 524.7 472.2 472.2 524.7 472.2 314.8 472.2 524.7 314.8 314.8 472.2 262.3 839.5 577.2 524.7] endobj 94 0 obj [285.5 799.4 513.9 799.4 513.9 543.7 770.7 777.7 733.6 847.5 756.3 656.2 804.8 850.1 449.3 566.3 870.4 699.4 992.9 821.6 782.1 656.2 810.6 777.6 627.8 599.6 699.1 599.4 970.5 849 596.5 699.2 399.7 399.7 399.7 1027.8 1027.8 424.4 544.5 440.4 444.9 532.5 477.8 498.8 490.1 592.2 351.7 420.1 535.1 306.7 905.5 620 497.5 515.9 459.2 463.7 478.8 371.1] endobj 92 0 obj [856.5 513.9 856.5 799.4 285.5 399.7 399.7 513.9 799.4 285.5 342.6 285.5 513.9 513.9 513.9 513.9 513.9 513.9 513.9 513.9 513.9 513.9 513.9 285.5 285.5 285.5 799.4 485.3 485.3 799.4 770.7 727.9 742.3 785 699.4 670.8 806.5 770.7 371 528.1 799.2 642.3 942 770.7 799.4 699.4 799.4 756.5 571 742.3 770.7 770.7 1056.2 770.7 770.7 628.1 285.5 513.9 285.5] endobj 41 0 obj <> endobj 67 0 obj [600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600] endobj 56 0 obj [500 500 167 333 556 278 333 333 0 333 675 0 556 389 333 278 0 0 0 0 0 0 0 0 0 0 0 0 333 214 250 333 420 500 500 833 778 333 333 333 500 675 250 333 250 278 500 500 500 500 500 500 500 500 500 500 333 333 675 675 675 500 920 611 611 667 722 611 611 722 722 333 444 667 556 833 667 722 611 722 611 500 556 722 611 833 611 556 556 389 278 389 422 500 333 500 500 444 500 444 278 500 500 278 278 444 278 722 500 500 500 500 389 389 278 500 444 667 444 444] endobj 54 0 obj [556 556 167 333 611 278 333 333 0 333 564 0 611 444 333 278 0 0 0 0 0 0 0 0 0 0 0 0 333 180 250 333 408 500 500 833 778 333 333 333 500 564 250 333 250 278 500 500 500 500 500 500 500 500 500 500 278 278 564 564 564 444 921 722 667 667 722 611 556 722 722 333 389 722 611 889 722 722 556 722 667 556 611 722 722 944 722 722 611 333 278 333 469 500 333 444 500 444 500 444 333 500 500 278 278 500 278 778 500 500 500 500 333 389 278 500 500 722 500 500 444 480 200 480 541 0 0 0 333 500 444 1000 500 500 333 1000 556 333 889 0 0 0 0 0 0 444 444 350 500 1000 333 980 389 333 722 0 0 722 0 333 500 500 500 500 200 500 333 760 276 500 564 333 760 333 400 564 300 300 333] endobj 52 0 obj [556 556 167 333 667 278 333 333 0 333 570 0 667 444 333 278 0 0 0 0 0 0 0 0 0 0 0 0 333 278 250 333 555 500 500 1000 833 333 333 333 500 570 250 333 250 278 500 500 500 500 500 500 500 500 500 500 333 333 570 570 570 500 930 722 667 722 722 667 611 778 778 389 500 778 667 944 722 778 611 778 722 556 667 722 722 1000 722 722 667 333 278 333 581 500 333 500 556 444 556 444 333 500 556 278 333 556 278 833 556 500 556 556 444 389 333 556 500 722 500 500 444] endobj 50 0 obj [799.4 513.9 799.4 513.9 799.4 799.4 799.4 799.4 799.4 799.4 799.4 1027.8 513.9 513.9 799.4 799.4 799.4 799.4 799.4 799.4 799.4 799.4 799.4 799.4 799.4 799.4 1027.8 1027.8 799.4 799.4 1027.8 1027.8 513.9 513.9 1027.8 1027.8 1027.8 799.4 1027.8 1027.8 628.1 628.1 1027.8 1027.8 1027.8 799.4 279.3 1027.8 685.2 685.2 913.6 913.6 0 0 571 571 685.2 513.9 742.3 742.3 799.4 799.4 628.1 821.1 673.6 542.6 793.8 542.4 736.3 610.9 871 562.7 696.6 782.2 707.9 1229.2 842.1 816.3 716.8 839.3 873.9 622.4 563.2 642.3 632.1 1017.5 732.4 685 742 685.2 685.2 685.2 685.2 685.2 628.1 628.1 456.8 456.8 456.8 456.8 513.9 513.9 399.7 399.7] endobj 48 0 obj [574.1] endobj 46 0 obj [638.9] endobj 44 0 obj [667 222 333 333 389 584 278 333 278 278 556 556 556 556 556 556 556 556 556 556 278 278 584 584 584 556 1015 667 667 722 722 667 611 778 722 278 500 667 556 833 722 778 667 778 722 667 611 722 667 944 667 667 611 278 278 278 469 556 222 556 556 500 556 556 278 556 556 222 222 500 222 833 556 556 556 556 333 500 278 556 500 722 500 500] endobj 42 0 obj [722 722 722 722 667 611 778 722 278 556 722 611 833 722 778 667 778 722 667 611 722 667 944 667 667 611 333 278 333 584 556 278 556 611 556 611 556 333 611 611 278 278 556 278 889 611 611 611 611 389 556 333 611 556 778 556 556] endobj 433 0 obj <>stream xڍtTo7t3FH -*tw#FI RH ( <9{vwu_a`, E@HZHT(PՔ@7  &6= D~Et@&͌Lo )$& Iqf1vO& wM/dzHja( >PC,/wC /Wf vu #.GV{U A0P  p4dMQ =\_6 .?&'lk%ptgwbR0 #í'I␀#(a f8 Qd,%DQd BpW_oz`0,"\zBGh}T~stjqN64# qS8}[:z?c!vЖ#}1tR ˭Y&T6T/kԍפwܔyk|xhJAZaz_fбȚL%xB>D[GeVh?#K?8x3*H;x SMu5 jy%;TԪtƺF d NId; m'䓴3qɮ[# K 'o Nҟ_YJz/SEGCQUK!a|,uAmYC4M7Az{eʺ%ëZ`ϒ1C\+lR\N1Gn5EcExӦR;x``"W=4QId6]:Bb>˦|$ iI^;h%|<ۍex.65§3o ٍETxRT, Z/D){S@@_[dT88Eˍ0U٩՞} [ |{`]~iJNpX0l""VjzVH]fh4p~ZWɄq$W ޤ0k+1-/ػtiMU.lpO}#L(ћ?&լ=hֻdFqFzͬ0حW{Br[Pсj=ٴ{j,3QwcIcjb 3UitH?9eܛTXi ;浽k)R>n ?X,}Ϯ{ϕ+1Y>Q,VDC7喝< gі7Rڠ'7kٍ̟xJ%7lCCJihec>)x կݩ?F!=u{  nG۞zwd s$EM})MtkSgWU%`9]"1CqO^a0]h[".ZD.:|7}PYD%K\~ا3@ M&ރZ)G1{C7k g_i%y!"~,Dz3UyjL3G%T :V)}rU7,Qbļxi4Ir js f8y14wnӯ"? ش]_6a.Ӕ~Q|V~=休RcC.$EO&ζTaZ\)x9ơts"Vc0˙!u|Y$x]HpziK[h]eI _7}#PUmS #kU4(JQ{v+NVu ;}-3wǑCs%W&3t.>73{=mxQ͙a,p>.Օ_ckYΠR>Og$)4cY\g05AN?k5 uh~ }(tpSs0^;Ol5M>W_j$\5s) \AվRª ?L,O@yS[ޗJAvٰ;pR80q1#I=ԙ^ov/M#sq]Xnm+,qe+ R@˜!rGV_: w}\V$/'5Bc Yk{jӝ[Qla)qN%2iQ x<<Xf^wC}}M>Cuy= Y>Zj4J[{O Zi ed-WsEV,R^S. VlVhVL 1u$>\+ (՜T"SS;I5hP/`%"0mNƳYj8Oa5i>y O\|$՝,hRԟKzYK45\йDcWnצΰƾ})9tIc'_ꓣ?K>cn^SV[Aw1:;Ҟft` ݅=J)k_A2I2~4%ﮋϧlTO ͏*[R+ڔ 8 ݚ;ؓA{d ϑ;<&{Y0==M$wb 30܅'7T(/46DH@P " Dz;AH`FU#; tdĘ=yqWtaaye*h"77C,8nVfg*my45 (>ֺ]YnL`m D{|=`z}MyS+񆞸":I/:AcƝOm"]_㲭5U93n" աrWfesnuў-u7ӆP'˙Fs-_~Vw^aV8dye}JcQ ?̉=ģ}4Y `YĦ֓J_aoMo#Mqڷn`\=K٬;xj%yI]m,wj!g݃_,c+~ʔӝQPe\yr ~oyfЬj35&>WZ7qPBиCN/`do{h."yXigL9^.DsQxd/Ӎ"Mbv HupzrʗlDO>ZJ-("gDHEvj@W{f`~djEbPJÏKvyYȲ oysЍJք> zl7&|݌a<Kn!׻k?0,\rR]fLI7{ x1hrR 9^=Ÿ:Y7#ξ*2wOyy&O$$nE`T&i{q&쩃q-0FIKWvC8_|ԾP† "X`hۙ5p<ӿmҀa',T 3p6:i#DlvT5? Mfc-2N^vY?)R&1*9f^ھw-U~VZMD5@jcJ?$ dM)ݝzQ`8^.yo\ 5Gv,ޮ͎鐛??"}m\m4hJk$]VV9s; ,a]R5Fnm`\ /2-3Qx$y<,bNW&%0#ϊ Hejuzf%F^rߘNë:˨ 9R Rնܹĩ掺27 8Jo}R/::ӣ`fC ?C}F wdF`e~fjQm 9(w[Y5_OO5P<<`#hsBˌhaRs6ԟ vEYCû7?ʾ -S3aU݋1bM^ y_Y/{:!a 1VgfZ1uLu?vg]<4Y^Ux\)~9L00XdbS~3fZ=L\j6=p(Lj A-: Tf#|U^y,їg-eVwV`/u.z:|NI㛘CpUjѼZYWv6'^aL Iu J6ɡX\SWxd~ϓ 5`9SgK/D3KF8mN`!X&u7*h5'hA2]7 &atp }, bl{SS1xDĝRO߸ON=2F^u:<%Tf>MzØ /F!+\؅q2K&SH3QWixVk=E;,-D.8l0b"V퓒 VjҲd=uH\N=wCނɦG endstream endobj 45 0 obj <> endobj 434 0 obj <>stream xڍuT6NwH8tI7ݠ0  1]tJHIwwtHI|ss~oZ3{1:\60k kjxyyyq 'ff##@ D<8M yys(= 6Mn 0\ v`DI:  D؃:a _%$qOOOn3f' Ap{`d8{/> t N pڀj]п5 x {7>n;w!O29h+ip#O@@B:dur py~yز"F "8SAkf0O= !VU;]`/=.?N }]`.ۇ![/ t7@@5O3/pn/9?>= ?ˣ`a\b.~!^@etO*W{W3ou쁶`Xn+ z?c6q!N<M؃j K`zU%B'+Wxmt _n[kN(X~$Q:m X{&z)jwVehӊ! (YםB7-^'k1K_$xV=L(^ .i>Xv}i{O^[JztiD+]:P}9N D10"(yوx2Pw \JҲi7X<|^m"_pS<dA̖槞Z\nj)K5bBbc"Q)Q!nUW0Tzd# رQۧ8Mop(-}V3fïDGreܥc [D6P:bB+xJzUF gc?戔YD NW{SJA,3/#C/NLe%/I{sPnh@wk[/DqKZ_竫}vɦgR1뢓A/`t\Pjt0 6[}%TkDoO0P'Q<bǕTOOK%x:}|VS;uDr+]^ORMD@,X>RnsQ،LXw뛖[g%x\ٗdly37{xWB5HL,rOdBe '9c9qn&ƞC fhN%zl˼ $ }0+?aA⡅MA,P9/n7UR:c(˕Kc4gc!Јɬ_AOo?&ܣ^R4,7\R4z}ls}~r&;,J\'Yj!o0垟BҼ5es9/CI Bʏbk$,AyfA_6k'Kt#|OM<;*iLkyRqt/z<C^G<=Bҏup3-Ny] rxqvI5V'B_ %4 8P" c_,m~:)#hRhT7>0l'A3Ď8%gu|6/B]nj#y yro{Um ukHX)^]x"u@%5 K\6gG&iv~정AQ ?:1CB6) N-ٗ <>]7IooCHpu݁b]^gai#zADc @2:`H9or7C0ʒKBAGP Tx8iIEv ?',P+UM苖W8e[)kv)d<gg8~`\7U4DE4A9h;v~|(ܳ'Lp-IBq*e+WkRՑm:Ai%(iSwxDzxA#gY ) k8qtNğDߕF# WZU(9cvIigU>wSghI[.!ZcDb5R"\yjKw_S *:&MKIMYm_5wLnVxZ\ϥHإaXqd^z m=s\q 1 Cw0{:avАZҫc[$KjX1HhʹU5{t$ߏYl9 m̠0![Ul7=mKF|؁`^͊o6X< %~ dT KƢuJpJQ, <ǯ)֖ƈ 5O*-F8W'%/ ׏6e V, 8,Wy&mp"FM8_Bm\ )=Ȓ/#Gvcz8uϕy@t0*jX685vfB,3FHH"Cu=Y&tFS~RNDWOBIU2)G?(Jy ?<Ԧ h`(GVz[!š/b:/ ni Bkn쎊wS* CϺwT\Bʤ'smigUB5ZbM*Qb,i5"[^V5AKxk:"!šIls3~]bmDT{JB(G =; Ȣn2uf+,ȒG *'FO]n~:}1 ~}.Vl%ds$ਤl.t)LJ'őֹ`,W3!ND#1KU6~hPg+.ZWci]f~S@ݠD G1r}z*FH}.z*sըׄFiE%,|qP5JSqްԅ<~^mjTJs$*JrӸ03㛫ʮiCQ>vL{bݕ*Ho  (-RBl>u$g0s3CQ>WD9+RAɏ#܂\dR l͚ٔ$LF28mr7$cIhyY\eD,LuTV?VC`R4ҁ&+ b- "6렎$7S '!i7 80? ܃wWv$h=1n,ݿ2yF؁{d0!x֚f1Enxfu7&qs^Ó(&a_pJ$E[獅j;_2]iRc {2kӥ Ds<,OrXJZk9 oh=uC,gK6lwEk}t= vWNy]-zXT2$MtO^hZd)Sg Rb4S[~Æ7}i:dCjh:ZZsۦbyiN4nVeG`q8aH 0MSx G o|EmTf 2hڨ"aIww^HRhM:y+ەXƠTⶨCYETp5V"[{v2nQ_Q9r qwf-ChiPp3&Wx"1vQ)c:Lc$sCsa%|-兀vx'dc!* H*$Wd{}Kah}|񈅅ͧ_%w$uc\a/;|̜soVVnvڜv3o_c_2ęUUqCv@?"W]=  2|&zصtq5P#OzȐ*wՁ@n^xQ?Y}Lј+*j@}M }TDS4kFΔ,9#,iz"X3 b hD;:3/>3"W!qzؗu..%^] *.HqU7f§5f-q=>vje}k9E8g.^L-j?uwr C02*FAaqU)NJT& gLMZgCDnI&K;ᢨ/Nc%X6M |.jbֆpOLC綾2hk$}0Y4q:M]ùvZ7ts'v_Q!DIj3o tX%̺$Sy?Pp<{4ېT{H’/P[\&c-(*S|hukޑ47YQNZ!mX"*vÌz}Fuw6vӏ ʸ.޸+5kॿp #NΣm329g#k;Oɍ寨T{_ȆM uWTQwP6Rt-/VY]k9@w3#zfYH6e].5TO,A: Rn:4kp>VH,fx/5Ȭ4П]1 qЮN B}tݑ0o\UywjqEȠY& dlay Kco4>ޮsg6)Prsh$^Q,U^'Mxڰʅ(Y']}HuaMʅ>Q= UѪ^E>t,5BA=Ï2xrnsjG߶b wX˪$&'] pr}_NE /lǭK)αB\ 4n)Ȳw'}Ja&aD3AG P1M6EO ~w+.UCM] ɼz\ir2ME:GRY{F5ڹEEn#մecLZ&9<zI/٬)?;㾉"`vCj ~ .Ypev+gJ+O[H,dYem;02*O2hj~(=hzNIȀ[fDd{*ATp_ uXF<ĖBrh&Hr>ݦT<|γFAsĨCL߶7_ShɷY Zl4_enSh@?cR endstream endobj 93 0 obj <> endobj 98 0 obj <>stream xڍTk6LK(HĀt΀4H  0:0t7$nA$K;q_֬5kǽY¨#c("(/P @ ?>  @NP\04p3 @b@ W )@U"ppGBmPwn sNCP+0FAN+(v(+/މBQvg'b 50@l3/> @Eؠ\HA p g5 ;r }7/U B;P- yQn(nn+ sB]P.w`|79Y!('^'(׈|ݲZao'EBݝf_pϿ (|p3DE;?-¢#fe  {:] ߎZ jXBlpT!6#n@Ͽ^'~4Lo, @H&|2`mW 5c 88c-Vw_gNc*7oC0o7o #3N;7GkzUP;!maF" b EYa\`P8DZܭ|wҲzupoN9ŸDX j: ȕgcˆ;dίK؏g=w64^z]y6ш?=By8wGc-׵+zO,"٤gJnB&7t6˅D6/gɲLbArd 2ލ|u>^ߩ91(v*gaiz<܉<ySʡ".3-̷ GI*j̑LZ Wم6_>յy^V9.}9 Il8[k6,꒢uq*ؙω%b1Z !*dg]ފ\a%<d/h@@s^Qޥ-S0WBs,RCسRZfT/R3Ǯ Nt3=QQؘ* 7 py.àT1F0Gq}7gpڿG60Sz{D֌+|5L7jEO8(믖g~NZIJ2kxɺ0F=XlͿr.Vga˛R*5bZ>4_ǵ~ZghS.Dna>6o58}wccc'ԴIg@\u& +Ԛ_S76u2U)!K=X5 $vEly\dq/W=Ҥ/랖9V.K<5m5Fg^ 7y%6㋛RQr8:e2>qj_$=Z/k`\m\5 6脃g YA5i `nҭDW^̠ү~>HR[H 'Pdbm9qӖx;z>SF%z .T_Ts}L֗CÆ&ST)J+GǼOܩ'o|YLL6h=qa4I1?UiU pPC\#~ Sh*~pMf~zXw^[/G7R'8LRz3>=;a[Hp LHEv}OI+8d7⬛Zm`6޲#Tu:񞿓c96)3t\33AW[OCKR9jVTç+RVΟp&LkMSܡYa =O+~^|gƶg!<Ιvt~%w]Y\(X_K9ogPġN'X .Qu]Yxv>+y~z>pݤreyQ5鹮E |Tw'džV緲0i~5)-dμ漰H޹יgsܑkfV7ЕU۔(!HZ,.t/o%`0m Wĉe@{$lK֩Gɾnna)s 0'K4$^}}]cVȯ$¬Jgd)0?NZ"ИQz~`ic/ ]U 5m m(Ke53_,{FBQGܨٍYܬƻݬ5vOen2ddFL}S'K`9>;f&kwQLqڂ I{ 91Q^zP$DYIRm"P<_#@m` >46FLW+}$"R2Il4^7-Q0vmY[ڬƺ≄~ӕs@誣A^]@ l'8Ǜ6{9[b/yŁ.-J¢*8iG+p;z*^VƎ`V UKHX6VTF kk6_ pCMŃ|qƳLiQΛ@w=4t_]@dK; <ѝtrhBhh*}KĹLs]ݨIWIh3OW܇ՒY̋89sXa8eG4/ea9OP݆o'.loDVW1xCXr .MtW!/RV/frx\Ul-Pmfx1oq0 vךG+y7/ǖ>Zl v4ө5_TZ)ElNr!9-z.{4~VfH2~i&bܾ٫v79/pY]Wa$.{K O~1"en8%k|təxBf)}QN!“ y&NllΚg|hD?IИI@ي)X (a(黕h$jQ[En:܄(IU8%:1g̢/X.8.__Fw>5{& }qh9e&V)QPcl/iTc)Q#K]3Yދ?vY iz'uq P*Mx=3ƴpdCyv-ZWk(MlD)lE 39[jtFɆ. A:kKo>8 )h I;;Kv߬XlZR[@vRP#FTQw+=1((Ψ 8HfKzĈHN6GC$(QMИ z c?fƥol;h ^C½8: ʟ8G{YCeO0~ q]wxp,Ze*On2̀ʋG~8S㣱b;ubc];XmT41"N/Ih""XZ/vai,$AZG`mkNHMvّWeNqք[!16)ݵRk*VhҺʬ M? .p4R4Z5b#p {ӚYc>vB_+|6K );kLJE ĥ[l6N Nk;#ƧX^v3*Eh)F1DoVYUs"1y^1`D.5]mqTixU~G!ݎ+L)kgKE癑 Gom;}^L]<:k%.EoAerlBŔ3[;!Z_ ZnGp($;o/w(tz(?7e2݊F`c͌j_6ҝOSoF)UβT=jϘ@)tmJYz>f8U3ޝk6n{^AݵvƁpE}I]_jʕ͕RUπ/P| n#VU*_Kz;pmۊ1]v86Rv".Xڽӈtܓ) X2&:k_X8lBb|ND,:r֚޳<@ Oq 7j8nuXdWwJT-:SOlļjWhȱ;0K=CAbVjo/;V71Ƒ\ muv>V|g -Ϥ1h2O·mW%6z/}E ܤ|(ŗvW6]0YCup%˾[җ9]nD{56NQpNBOr"}$u["xGY{r.1,$U, ?>:K57~633|ZoP&ke1(9S1zdv\9$AjtPW~コ 'Ƀ)Sfm#De;G>ObcG%w9>%)c1 KezʇR!0eH ښ?50蕜TzSU2yN z|>u^WL >^*P ! Kp(۷% 'w[~8|W":5bk]MW-o'CGiڡg}*}3a`Sn`g3FAj<=4$%Wy|7"z'w^EI[i+N)4hQqqN!NyUA漼e _2=Tfy1 "3WqL3,֝a):\䈽W֥6$tR5UP?n:a;Ai|̲1^nkj32ʲB<1&]=v3#phJO}f 0Ё 5DFN f95J>p#+{^G%A !Ǩx[#s\ 6 aGPVK^pч-RTXF/B,wzamx[Z+*8,CW}yegYܚ Be8A!dHA"7ʐ=,"tZC'm}Bt"&l,7D|00VuKuSv/mJSiz2CBAݬFxOcfe "ġ]p%zCS)҃=~.9'xCB)`p&!'1n'ky_5C[8~< ӣ GF,zVjZ+# A\0!n-( 23U-v#0a`M3+PH kap+͛\I x4Cno\*O["R?rø ̕L6]N4fOmn4KdP%. %h\$J^R4Kf|׶Q*2=a }O^+:XyLasZ2 s[u)Ҵ=o!!31&Z3p6;" ~ W}Y{á ܕAf7H#9 endstream endobj 435 0 obj <>stream xڍtT[H#H 0!! 1  1C7"- -Htw}k9ϳ>{H!m* ܜ 0xua?(>/^ vfj8@ <  "9; PF.XG/g)Y owj`W=@A](f( vpD8[K<`6m j U.@]'#@AXz{]Pg%U#X;9@0og0ppý`pk PPtteᖿ .{;f78 Wu.g W\7Yn)pp]]~'sBZϟ niK7G.=8 $=f u@ Aa p MrqD8KX>.`w( ﰸ0+j cZ߼3` 7^[?/멞dd^a? /E + ?Jp+@O]O>`;:^P? ovW;7{,`_^\ﵯ@߬+~7Zj\!65`08T8A bwh7) ,M º?~ - #\]XSe A=+_o'`n.}/X\ep9޿C%4pws{oW?B.o_-9߇s-~guB=D4ض&J҃csD|q5 ϼs.z2Keftr'y^Sz؇$kf1O{ҵ=ԘTR[NvMHʌo܄p5.I3V9 o?h'r{Lǥ 958N2٬OIҧ9&B%|Jv}h{M2 FRb3gz f("j^;(Nr˹G+TڟiAjg"ؼ9߆ ^+ڮ_[Qܐ*=s20[毎)^40:K58׸ydBB;1춯O~K4͠qB`[sS/q kK(Rj'?)<ϬBU|/%nuiA4 *q緱ތ9(ӏUԕo̻L4u!ƈѤPߍΠ4onYJrV9|&6~穴ɪE˕[}`OF@l̵#C [ O5E:o I'5a gUY5u MoۃKdJTLR9[_7h"8_lzI?x r+oP0o~|Aw\Vv{i|Ki$k,sXF;t|*&>J#eB󍽔l5%#!=Lk|P"[gIfVDI /p9Gb5& q$Ndz6 é{NkM"uU{a3_o46Ō!BcJl̐vdה95qNh{*;F C_萎i"Kҳ!pYg(E'ZRJ^k+I_r^1oM,w*=dW2 % q\q H4[ODžm;g2&SfE͛˚,:p884My3)rM j.x JKd[ 8Ugڇ~jw܈+Y!IoC<9!ށZهA +z<3z֩J-u_r&d}+~k$ W1cg @u&&?oŚhd9Jhҿ׏^.p(~"(@m$l'q9=7LRSWCڙ|d(5h /P}6hRXnpUe.@g6~Iu׈Cڌبp~n(_A ?.c%ho Pln)JI8s3ه#B%{hhGjЁU¸N]G<|Crb@acr!pz֣Z̆{ԩp\ADSE#wlItˊbm;a,U{^ӄ'*r5|_G=lҨAX=w>ˮZjFӤ>cBط~0w40J\7#z碂O?-bfz~.?dA{zZ f,W'^{}#ie}iX DwD&SԠėY[Kj57"7y[P0eZ,W ޣXS|axsj*m~YѬWm(VF387ۀXkcWφ>< g t=ZE5't,iƙ+L_:3Hႋo_O n֪I•xEGbO]ثǶ6#ȳ>}\88 =Z{t ^Ѩzx7Jƪ{kS?pbPO(qyA:hl}dRLx ϻM:, VsW I]Dex$\ v#q 5Ծ>xƃCYȤB>4 zs Zw_+"VK l?"kmɊAGWr!Ndw強z7zn_͓SmRl>k!;t^]lMݦ'3ɌzBg1lA&HyXK }*R2e7;=5;I^X]]2`tT-\*-+~d3Q6M*=ﴙ]r4gВ.w}sR`WJ%xu:DZ4u4vύhʤ5 Ta̾HIY+PiR9 s=nvN-5qV GXdw-I1!%)김$F5Ta?$яActpVUj24"$1㱛O!7V|ob?kN' U 8K4B\OpN fָɐv.:|O]^!YpliţQƤʄe}2܌{Qgz=8\l|Z]8OD1^gxaV,D_Q=JxNahXIdR)+"6ܧE9VJN w~cw3vny< Q1)˸mU)ZC}KkHʛ)r#O:-H gDCԸV[vMvhD!Ic PG 0֋2s*~YSNc3zB=rtmP鍖.w;[X7h&dPw{܏mEr&?BM eQԾ[tjN]bܿOuQ:CX@anAt>gusxeӸܐQ&]$أ|xjY(nA&V:3J66df(dn6Ig.Q󁹇;fxgiB>sv_YIlpߒ I/r\)k8m ԔknݼH{GG_#Hq5=k5SqZy 0jkKlRכEY:(xP'=08KŝƖb!ɦq 4g<Wm>R՞jN+n_w)ulͲU NDOq9~F6{w֢bHTwV5뾰 R̉!Te#D DMu|4}p["HZ[M[]C &;yYsw]1l:Ыd8k";G]8 b]xNsp+B괾iv,[}/7C/m$H>l]f'GJdT(:T&"wa~~?;O.oe2PF*]?I_4]WzIԶ3oX9OL(NIh#ڡ1Ek֩6ZE{oyt_y6,vx!59Ҕo& %t:Qe%hVeҖQPI2h3 7%i2 endstream endobj 91 0 obj <> endobj 436 0 obj <>stream xڍTk6L# !+hA?nז_OοTv`FLXLB /Y~tBKky ʕ׍KBB$B#k&o6 /( !׎BnK zZo~GAؿ ?gXÛoL0CsM:`+Iӝ, c9zؒ4PXTx C1HE.OԻEjecΝ`"TUdJFU3M(8&cMeDg7nbEṛMQ ĊFĝ(Пy/etOOwe}HV.^FIOs5':1{7 .cŢdm/zn[WVzkX&бVa)Ѽ-6VaᄭݹL%_Ytmw3zfǤx)̧ÕѡH|X]}^͟r=8${ݒQ&\3zzMpnI0N6K@J!8&S%n?&&yn {@s@`\uL~Y$Wmô/"H ,LҕC(d]Q>g7eIE铛g9F[532f ),8zж#Ă0/=Qk9U)8}\%aSOv-9hE)͗y}_?tXZ'vˮ5O-/OJ1)sV~?1:AiyzjK$#1IXݲT][{Co zs 2ac>nȦ4kZ>F„ɭf;?v9A'yK)Krwn3|8X0 b7xZA1IN-yLF]{.?sISZI]ԇt%jZ" =-!0;>FS@AtƸpY-'f`J5T95h Zh'Pb ,;P(ǻOˬ%ݻ-_07q}usV.pQOo }P"9[&?nxE=SD<qehc\13 )x;`„B sx;|4p*!T27-XGo:FB2߽/4++q$_CEڿ`VZoTs'0֜Jb7A<"1wXS䶩f|rpyӮaTt)yL$eN8Kx#1&kj#̎5k'muwfO{l ]!d.W7?bjty[,zՇ$rޓ?M"5&bDhzM,6kA4GPmղ18rRJAG$:[.pQO[Ƨ *U;wȆc|椿$9H&!0!>,y42N|~WZ++ZhM۸g S ܑyX ^XOOq ߯߳n9, ȥ/e~̕{isZ/Uaz1!zlD.;b^cz8&|i{^JGO m|G!߱3mz҇ey㷯0SFh%hWeg,qni"j8.^c!. 3vLOFt{u}M.D|tk&jL~ "eߕ#Lhv"/lfՑskF`s#JT\U:-8VW(g-Sټ=_vStA|:iAHPEp"y>05J}H 1,hof+˯׮L??;|%{IoO|.ӳzvw JŕRR00R=~K\05wHzXFt3=T9b}yY֩u>x>T vHI3vG^D(5mN suyP;~qӦ†.Mm7pKRݮ*0j嶇9~yD]83:jo;!8 R6O)+- i|b0- -CFnY#~Y<-񂮒I\IoΣ=LxX3q5z-$7_i h'a橮+O+@_欷׷#{,l]hD W[:j)Ccy]%"B>'*r0ק`3trX}n:>5HT-]]46dX;0Q|nQ1hz֊^dAi|=rDFYwj3-, 74zI:,i:{mpD8w2dx2l6tc#8I"shF]?FfeT kDhǯenObK~}tqduOe l G-[yȡG(nzTRlce}ʉj_/^ ͌2C^⾐#f hb5bQw|jei]2-)7h3бm%äm;α*TV-t;n7Y^wI[#t8֓4q  ݏt;@}(*gYhn/ikޔ"W=l>[3G _//Ec7Ov " +.w ftd[n0bpԽq)?XJׁ2ygs-%=Qr srԷ e)1ovC;!n?[*"GnxdѼg:cWWؓJc쎷6,ׂ!| a|M&q5ٵ a7+nvHvM;g |>5g=d'@Gcl2/&/;LEZ"݌qiN:)N[p6(rӬ'H.,D9^IfLTNmBRt:͆c,]0Mu\GU,[Dc2,Vǝ%; mPT~븞1,.(eo*ꎖ)Ok\wYiϟ>vEh~RDm/vwcT>3r\# cfg(džޅa< >8S#uKxbsNVki9@b^uIB8cE~y>{Fx{UIV1j8(9/?dỊ\ڏM?Ԙ3Q>=G%5=2% ?gu,C:|-8EOnϤ5*"_԰{LHy!Œ{8hQZ4kܥoxiWkumbgQM~k }߸|uISʨܛ?S3iQzٮ*K%YAǣcm?톪R]?xPInYC:?U+@a?E5njd9whM]|虑[0(VɍddJC8?bչS4[<,k `PѬP/e?깢UTrEBJ&;Lʶ+^A5&'D" O? b94Pv6$vtXv}Rum!_O 2JB*[ kRʔ\C>uRN^ؼh v-,̾`X>Ʈ}G_ԗNc}xwln(P0M.)R]^%$-QN+<=^+3`eIEHU1 xeĚW.:.D9UzGQ]PEvu?9Zq>-f-=ʕzJV<*:a1Mӳ(RCI`C :0lm~b9+@z+(}$Igr\韜u/Q,u=5#/? CxY us|m5Ef71/F$:9kr/O j]t*[H1~YXJDF-k Ռ~pˁ _wϿo#,\vAS?kޛUJ԰}azSV7~Qk x%.Sӵ "QyYtRR ߞCt՜+F /gG8d8o`V9>n06x3ҵ\rZg1x$Wsc3H=ܑ|.m:92ፋtݬ='UR?;ߑjxv߷%2f8Nim0Lg6&C#=^nT@ j2,7{pwn6H0tXa%vV5u;0)N2k'aBa߃/)NzKs<[3/OyKe'Q8BI^f s`@>ψfWە8NB.9S,e2ݳNg[3:efmV;x#xYJoh0V$r~VYSgK2q_kIzYɅ*ka ,qò~p.ѹGJG8luⶥh|; ȋz0s3"zM#K~0'wwZ6x7Nxꐫ6e|ƏjzP oYaⲊ8򆨦ץ^,/'͖R pb4j4oE.Kqg3ET}D][9zo,g$HP-Ѵ[(hPq8`RO{T+}yAG^MZJLAz?͊"1ϔXd-ޠ34թaW 7 t+b,LVfhNk.B]Mنd~eX,w+]D}8D,zuvӛK|[m#{yV͇Jμ. <1z u*~T<~#ne暑{{UPD̄1m"`luF[s?>FL4h.I"X<͵=N,O G"ܝuq64̯6QTV7xNWaYiVoSD$7 "[UU3OS '&q7UJ?&P6h"R@ޮȩ(pp]f,S)nsN#֛w3|_HŪUtaorPs͠.᠄NH44SԦ;ұy_0D^?V-y~?nM^>O 3yb\-rY.NQfإO7\OᲩJkV\*w&XRohN[Y@3ٹB[l"8Afy?IZ, ZC=5j6 XgYLy< =f'ᢨ*Qې qH1UJD2Z^VJ` endstream endobj 367 0 obj <> endobj 437 0 obj <>stream xڍvT6)%2@Ft7H 6`6`APC:DS.T:A$3=}پ=d5l`($F I L &4ca$074_7)B0X PwwE`qi4DI!p(@ PG!ahN-G-/,%%!; m!HC`+B([8 <'@Qnwyp@y_!؟ɀ$C8eX3Dc#ܑP[` q!5oDp`- AzÑ;3 xa$Fa!3 9,`; Ap_# Je%$T@4ɯn0[{ ٬{#v!05ſ?6{ \0/[_ ]`_f.(v?!EC<`;;y"P-`#Ik9c@>~zt~t ̌Lo߃(/(@PJTKWa)8Yt!]C45w< g.m08nbL#,/wCο<;{`9_ U&?ՂAa X#\Ap2 Յcl0ҜH. jF@ @ciêu(/  nnoMbb_0VPo"HCZ+XX ܠ6ο|$o憕o`[0̖dfe+̱Yi< RKxigkꥯ& Qв)E!#~L|;C^OU18J_ 8h݆nH J=Nߜ;d@~6-!2^9Q)^cԧ,('}jRFOehVkd:X-_r7̺-s{B}(^.W1Mr~ĩRb30EYm1Cn p>^x5zux &,ɩ"S|3쨈W*gt1I\ Ԝ"GЉԄ0*a_zKEcN!΍-)t(fk Ĉμ*֒UۤͰ^Nm#IqNi7;6ƓԵc5 avR(ej]apwC tl 4IL,o+h Z^i2tTŧU:c t>澕 ^ҵP\Y 3H+UҊDIa\1g6"V4çp2p .ͨҽ<^`=,BvctR8m?M0_tb!O,xQ䫶o0?YWu5̝Gwח"VO%m?]g^?,!UȥcF(='Hq}+36w>LSbgϞ@4@3s[,m(d as5Ch; Q$bbyw;{Uh?k^/+nx;=1i VYrtL$yI7ylL*#g8^o~uME(p_KDž}_i_ 9#z$߫6$|Htc sbUgIQwwD3l_LKM *ݱjKS߿Sg%_qr[^LYOϲod;AZ&Pbs;9i/"Q\uMx&Q<'Ev; s0|YP#dmsxtͽJ,:s]Mڢַ3xoh޿KS[yVf: 1 sӘ|'qLTv "6_z50DVh֛OO͐Mz3$VcZO䠓1_j!/v+ē%/ό|7K}笶k?n^2;vNjA(i*h4Ivzm*?D1Br$Y$ r3$r5 Aaf ʡ9U_⽃V7>軅]!8bdv#/Б9puhat\>k~ ɣ}4JC'ӜǠčbJ.oU95xV5ϑìΝ~dKJ~q {RD%<x!xs6@+5pXgMUXlvej]8;h'_P޲ qLm[&F@>᣾=,0 >ۻk{+;gJg8rU_:(d֜c(LF6 dwV~4hI!ّ̐ ܈g}IS,awj**T\gW4<ڳ=n"Ls#.`XSfGRލn9vÇXQ3eVR"ԕStG%D #;Y;ǁ#X)|L}U.ә@I P9[Cx`F庣Gٺ,ADG!蒴3x]m"Jy![& ٔo'Zg±}]I8Wy2äa\ZprW&ZtHUxߔ}gPYU(tXr m⌽4p 넮ϩp8Ŗ/Z-,ގxl,w׌\7AV޷޼jpO `Yt<ScfFXc@#oLL2*3~'BQxgNM,hF!ȚCz(a~dc$Ĝ̥*u;OiWڳJ?k>@4j&Xls,[^lyiH#F2puϏWIa'.o\6j"8>,̇曛:nT';p>z ڵǬWI49y 罛b',gjw 轲& +Kb8f'΂3G/{^Av6C)6*"E}ÅlͶN>h hZ&͉U+<3̻6O|GZ,)i@ƒSx2XS@>se\mjQ-6uPJ*R߾zn4).Hᅯ(P[1Qa2O ٽ<։i?(NΒrc-ǥj9 #|[t[}&+&^o/o!u,ͻdh/lJ.'i4Cvxi r*:D(^o:GWxSYY٧܎WŎ`Bsһ:BF92I+CC .-!Iz|CZ\E 6Qu9yӳcx[J}L_k/8&vWPR&жnm!@fB\I4T6/+Y]+_7]cx"4dw>+owNϠ@'ͮ@\'f4 aܦک9q4 ωY|倄@ WU'_Ar~ !My}W{ma6ID! zF\M\-g?Yv{S#~>gk\` RR*誇V?/݋^t)nWE*0fVwC^,3T3Q77Dl>[|~cMI4ߚ-@d F=퓻QyL O z&1lEY"?a<.9g:;yi]~ޓ3mBCn({qޕcOez&h&pb '@@F̂د#}n/0_l"B/HI' ©؀}j/ԾeD\3e5}g=W_e{RܪWyْ+c:\g%O[Wݎ ˮgo1yǑ.mʌ=-tM#sFUWM* Mlbfaŕl}wpFh-,>iܑ5ys\;!q*o'K)Ҝy5 2uu:}Ly!p@D좑`XOw*gתƛmGZ3Rs!1Y,hu N)*C^'[E1/B"E.WXoɩ'KffpJ 0Σ=>u]#gh:Ot'bkE&W}*8&QvFfͼȘmZaH?Wm;[dЧwg Gr撨!sgrO>fXjy0E.f`1u=}']cU2O_\&z8%tk(Svuq7DEff{*nv>qTfB9qoPR ,wB<6^>xC 2Y'/fc"lLP/ElMu5MW<$"+ʷ U{>G~ȵѴTJdX,g%T/u4OZ18[̖s8Ȋ60Ң5m{4LA\h#s endstream endobj 47 0 obj <> endobj 438 0 obj <>stream xڍtTn6 t IA6`06F ! !]"J~9wvζ羮~И_h h~a!4@HHT@HHA H*Dclj@4t<`aQ@DHH_DRz"RE Nh5 KKKv(BP A@B>+F zyy ]Q>xAA0W}+Oe\'("!A( A0unoo@Pog pu}pGh | ' c34@LCP74J*QWL`U+FO 0m3Y8 ~p4C= j)m4@\HZBB\q@AN›A~¿̘ nLCzBh$?`( 8B1C1GBB ~} F^`o ZSb**o4_Z\ ,,&3!7 j?b= r3>Z"($.| KMῢD ix`a]0h=ИC`TsȟՃjEP;b/,& $Ei@!`C(G2즿V C (请%$_f@.oYޫ!LD\D">B91 xV2@P@c\$ c@  :b@Q 4 r?D< Ka1WcF "A0藈 `zP7Llh<ҿο_"@dCBO딙?-tD&XGsO9fj؁NN39~ylV.K~G+8,,rr7шϚ'|2L[tfj% p`OcB*tlD[s.,RQ!=LMpq'*1 g>q߾3у= |ush봠U&f(v ŵ/x*OԣQ_5 CG#,*KB Gւ=ȣR{@b_{ݯ:?0HyId?ș{0 h;P xoG9AiL^C뼒ǽ2Z,-Q%/qXZJw.{n}׈TYjIHgKP"M_'LJ?lc:7043nK]fnEwxIɴbG+6;b곗va%Js"*5.KhgGxoujns񋷯<6_~Ť5ytX:k[|Wͱ e|~>2D?CKMք9F^-k<`Ƣ^+l>==9ʑh̞5[fWŝͯ5@8ཿ0w#x`9'~sd)z,mnV}h+@UkWvdj.\Mܵ 1_o+unM=j:fߪZ]cgR~_|m<Yleͪ0VslS)KЎ* lί?M׍^6usȇ[k*-SN\S|u|g4AUulRpFm,Y]:_@gm.pk8,}Crљ[r(AP$0N`9_;k ^%p^L8ٰͶm9,YQ6DP J?5tbQQ'7K2#V9feZZDAufd5l6O'#j`w=h>$ePSl.hϴ!'.Yg' uV-kغU #s1Klh^x;N3_R;4*eA=S_W⧖j.ZE>HLo}Hk5bȧŁ[J"19F].ӗ-p G#nӺOШXQW|#ʤkh2dpƟETV X*}>%?kf c_v3}j͆l47ՅRxc|چFP.߼~։~KN}fv:շGm-}t@9]O u6Ì+Jiҹ@xWZPjau'n)cw{8+*IѾOڱJMLoT8vٺ`G44 Mdf=z(Ei]٢e5GXkET0B\G6FuMʂKU~[qZJ;}pA^5$1> ) ;Nx;/|tS(-mG{BdT*Y^p H_ sO7EI4B;՜sN .wU$&** S]躖4xL뭍㙯l\z6@**mt#f]XwgphPo "$ڃinN|uIW…zť'>@mj=v/fl|7TKEJK"̒P ٳӃʡ̶"Ӡ p4IhG \'("a|3"CN+Ǹlid&:gxKu? UkRyRrv4^Oz𡏶.o`6kD;#^b>LIMAKKhd|WI.PhnV<@_xdr$ȸ辴~ Cv XBxVd!21m<2y&ki*x@<7b{Bl쩙ӱ \xR.mr"7g,r3px1oɢXp5ßG.$A6=Μ]v/nԙ9kdAFƝJrwJkWn* !ouuFzfUzmK230}RnP8c>/R}˲lzeВُK@:M͍*<$rGȣKoM 53W+4ٖ;hj)9N%U})COӂ|+:0 {YA z#5 [RjtY6fVǚ+7G1ddHir|&$>kנXX.(N;#A_YnR:*v(/2MTPhƩ$1k+xȭSٿ{qۮ_ẅәG!EvF'O Fj&_V tR@!f bPР%2%|zBRktuA:rw_?uB9/tVK)U3Ad_GbH&+a'L ˳Fx^Dҿ"+F`w̸ב~ 0M֜aѕ b=z^vE+֢,_mL*rNRsY|P922[kh.s%H=tQsW[6j?)GϏ ?+DDd7{{Ik=Gql渹F{HH59 %)(j2镍mu'FS-g+,4 Gs,-N8Dkq]Tױ-`b\2W]}"ˎhU\ka[`p91ycfKThS#:/>`>~b*Pk!kMMtZi -ZtCT1:hGw=+9)2peewwK"*JEr/0MŠu?no>J('mQ>L϶53X$e>#7.9^YYʲgVȎz$8b L 8(cR+ǹ'/e-4&D~'>?Sbv02G!=Ǔ~x7=+lde s_8un_,~j̘/%AP՟16SC?Ev䣸?@i7hvOit&̈́5.Q+HPȓct>Eǘ"Bsv[Sݸ$1_>ٕQmG6s>8q~6 vW7x'NnExl\e禔D'P]`bnmr9 KMiLgK.'И/E ?t_%R~^tި|-Rx̾iB6'SasD&IvjʔeHfbA;E;*|^ݟ{\'"xwO^(hS4[L*+y 4N&tv )4喇EcǞrI% V9"߲>Fci=\7T/tƹ=JOtU8}nSۿ9kB[%X ؃ Vi'D9b\%nW2q*4LߌuG;FRPZ#;vvs?VӅZՐDzt ÈM՜#nZS`[FvF\DZ!E+Q;"mپH@xna xHzI>x&i˫&YK>uhk9kLeyÔ$eDzmd]~\a-0F77G)j ^P$V4]5S#`җ|88<+Wٲn \o27Cy?}P~ j]̍%aG?xu@3סkYXtxN7X#n3HN3pW6Tt4RsuZ €{n\oЗ7_E ;@;wS/ݕ3 I WYt_+6ܿ 并$bkK ]⼜#ܨha$9G%'gv4h"n |fB#u>^O$'8 Zgu/MoP< dqa6`H+f#OQEo8j.s52!F#i)c,VP$c"[UxE"+h uL7" Oܯ.xc ފ2n*PlթRlܼx/mLIp;l|-xxB6eId%.l *PHT^s w/ K@̨ѓjjư1D9 ѐA{O"3*awqSy\(Bǖ[ЦiId`9mYʀQ,BQ԰"Gx^j/]BV;_٠O OaTjWs'O,Mi.xoP%YƎ*V=Aى8/)6! u'lŜt_B\~3Y/+{ɁCUD-#TХEel哼}2CocLjP(ڹߛs;oYWO9m;n T™mLo_ 0W tIR l;'ąٯHҥWp˹L$;ɍkC5K"@U1< RWXx5l/wҵg]}q:UD}%銒u3A!9Y{"zņ< Cw,ޡ:6-"Ev^;D<=ݜ \r!ARӚ߉&zZ}pl:eGuM x;ԨLF:tqp_:Dͧ7f ?RY"M2^V˚pd'ʓpkojU(U2хPwC/:+,UR'o834Š;9Υ'lHd%СGf߿bFX*PYpoZǃ̱2gssA ˉ[  [IT\+,kd- 7(*D\W}(BTYJoC֮wMV(VzUJ{H:c|Gݓc#@Rep~ÕhfPS*KE'Џ:B_eC({6 H 'EH5*ϩ״Әoe oNd!k]Rm38"LxK6 Gv&yTWӡ:%2qWP=~SJ>e$"xxOg-ԅ+Jc33h0 w^75g2q2l Ozƴk~taXͫZEEcfb,F" o:=urLB]nlLQ- 7Ʈ|V0WcG-ĝV<’{WWsץu*ojӖ񚔳K ڈi黣WE >7?ek`YTI!YS(EbgKę_u4Vtq_%/dya#X'^\˟,ARg1C GtIdpPcC?d endstream endobj 49 0 obj <> endobj 439 0 obj <>stream xڍtT־HH*0 ]CwHC3t4*҂twIK H -{{}k{?{yʬ# CY($/W׃xAz#/7!@! b} P5N98PD*P#`u^ Gʣ\vmpXsbb"ܿNpW5 PbN;ZC(kv8/ ͋r} @`:p4ug@l0PW8ሰ#nHp9@t# wB 'CQNPi A8`5^'E~hu>pZ]~wempƠy_#*s}ʊH< Ġ p[_ߟu@<>6$07g>}$ Q r"g EA" im3w}=3`s=a#AC?@ aXmH¿_6wExL{evM/ 7A :j?;&'x@B PP rg-(6΅ mP?^ӿ:vZkܔ_?SwῪo$nOp pMZ7̵Q2@7Gp鿣 ZHk2yh0B`P_H \gWZ_Cks_E5 Kg !aEx}זx-H7|H:p=JZ|ֿ\_6A>DE|dC6ђ43ٿ8nM8=/mQ&Ku$H}uoH㾦,/yz(=%:}@ ʿq0`7%Zte#2&Z R [q$3Iu"}]I--^CѷEEo6ɦ7Ƚo<@ m?gtg )t$ΪG ӪZXW9C8,僕X74Ct> F{DHeR{=})r߰Å6J\U+ {__ʫIK3~48^*(xox`oPCo&⌹\rZ*Շ8>$\R؞QI~ryĮ&déi7txK$7#.FNO8Z^5C]s gg3Up I*jj1*U?~X,8{x4玦N q&׏ aɣ 57d6v:a*%%3<:b S{JN>}U34mf:2Z2<{*} H1Λ"P~; OMs?1L+ٟ\WY)LP'p,00 PNk%1cw_ N *yHl["Պ0Ԋ^/Ǚp#cuZaI&l'I7>^V|Y^G鑶;R, ;ƪn\=ð/u$2Rd#)=Z37WW܂}y8iaFD ٽX9D8dDfHIz,"ﲽhr._5kVžj_ *de&RX3S[/VІ oРudj\8feSIuK}K$BWԪ< Ψ˪O2`hn*p/ϴ ǣwgNְ:GivLE仚lZ6~}5꧖'[kxnAviuR]nBhQ>oy#(Y݈h-{irOY)K8gi3=浱:*dػU{xfvGέ;=,EzVL5W:F>Px/ެk/JyT]ue|vWilGn]Sy5A9|/o qsV }>ڗvዕFYg%ٕT|eu D^tydG'P\VL-:kpqGkLUO!rI-6.VAAU{~HN좧AZ=>ϟCW:y!MY8 JTL6?B𠠴1JRRǛd`U.gv3$=JȠK\홺;e|ִH9 `E)aZ>/ ~H~:hY-L?! esmeoCh64oQ.tZ?&Jh5`~lZD-2?"j(V7N:dž<<"9cSn a2]F?Y7! mIu=0yn4K 7DSBl3:``el*(_or-38,-geWGqt%_we/ w=L|j$KY)N aJlBnAM `A4%A!><8P%s,~ŵi;ycj^|b 2O8ް ~?E(o֋2Y& gnhT]Z-!x f+mYcO .2AիQ@PDiETwcWjTBdnlwm@fSn`J8ԌF ȹڂrz1HuTO;v$M2 DŒxX8.|ώ{O7ŝbLzZb df|EՖ}M4;9-jIze"a= "7+Iځj1[n8ų#LȘaO65:MR&Ůl~'r%=<گAl/UWl^u \7cepQ~%)".u3@_+((bXUwux_l'~sե_u[ :r~ TVT Dm ["+ DIX0 |D  6pqv<>5oreҷI*?=؁ea&O`? ^b;hUޅ;`Zm8`0UqUIlYv|#Y^Q$!P'd"$|k@j#5 'R"OX}J-!ـ N@Tb[{EUQ#' EV:,(?p+ӧe)DŽQ l9p/mF0kb/a1_7JiKv%{~sP5؝  =mI VnfE|`H3˯d>Hεc '}E%gTw˄i%-uIkm/cz{.<"~<~&2Y%1Qz=*R3aݪ8{OT|k4bV(+M@* 9-^*5?)=)K+dzMrZƟ)}_]7i > !4'#gUm2xs[8Ĕy+,Uesꆵ`+)"Y&}F{U;Huy|ɃĄ 뷣eGM@~c[Dt?z#%C̨gK°+핥=OU"Q+'Q]pNouK5|b9S{ QNJ |<]yq1M kĩ zsbrtV@zo䩂CcKߕǭ]d˕Y; m0^. mʭmdV679Zw~(9v!]F˔#@tgKؾ' tb?vȃ_rS 0/fHi\UXӣM &^-~sj#y`(FYUĒVEwTcʤqrUYD("s/DDCu"Z~B囻|8*t808=>ԙ`}Пsٰ=JN;A jWarIY30B3gv/z%F1G 5rfQ=E1Hzԇ} 98OlQ(}ƛ2B-R2)T:ujSÍLaW47چ [I &kH,R"c5;k|dĞ>k:l%-4 5M@;Gǐ4}⫷VgoVs@I>5f +,/Aֱ)`]/\TVTgcL IG ҒW=ߢsuzk/ps@R*m >>9"!̳}Pr{U\Dqy3j8HGiS4eݍ֊mǷS/F"l (J# BT^M9fo?+䣩]p<%<\uq'ejhsR O-j>չzc=;m=r$ +tz3Vy#yYc)VKz.^ɹerVO޴t6Y1Al k pJm4}SZ19XS8p L&ʓy6ơEH|גo^߈H=k&Kvys#6SMzDg*ގgP%?/ئ+R gAT6I' h}*kӕ5U+_ ?S@6-=ET$M~3RlX'/ cZwSNqE7OuftutWϑ"rZ/Mt%&KIIv1_|׈ibHY6Ɲl9-%fTg43HPJpQU_Q+zW6n4*t5]O4mJ5@te#2:07+1Vy2䫻LgpES=j݄T,;#gKaLB$1 U3O.]"k> endobj 440 0 obj <>stream xڭweT]ݒ-\n%8 !w,r;'#WwY'Uoloon hm΄#A5nb>W kfh?HZ{̀̊v)4;}"$oOВ.֊6 q1ƎW Z84CC&AN w2`nlѩf@Gk-Cѿ `dea%_ hk?D:$ީG)haA?JQ3_vOƏ H7 ? Ύ wYX.?FYQs6526uqtPQt hkΔ/*5#͹'gp\V z|*n-r I޷V#]YlkdY79mo>:U;7?A rڱV&.A d;#=k&ŝ=iJ] VZFM1Uu@%t.}v <%1OiM+6JZ$qbK3ɹ΀H'`?w~.eڌovl>&XZjf& CbрXL9Z.YlC1A\Q\r@c9C|5PoG^⠲ETK5.Dy5#Q_clD?5vSXO;j=1G华GGq>9Vtp`*fȽB.-=R/H_XK-|J h,+й"et } }LUƤoz*UD|j卭W-=H`,dL6i4ȟRd 50cD9EUseE!go:Byb4y3ޑ#[Lף,f!~(~r2q`vƀu{d>O4ӣE2_.hZ8n:5JzSEXWQ-ڷ!L՜΁.f3"v/{J7?`1WUn.AVP 7Mv-+XW]-7&v_>v66&/GI}zQ}pBEJ)6uK7sbe~$$Mֻhbt˫g[~yh#J͍8,=Ei&vU!&RSY0́\ۮqJ^ c< 'lV k@.uxԞS}C*6#Lj 'EB "pZK 81XSvxt1Q-;<$]XGgiY_mKVe-QY?#eQNK@lh  s;kqCm֖=$<}_5VZ 5uO/p SJ6wx$oVrMȧ F?fXڏlj@N94Rd}Sda͔scjW|і!q#Uv)qZCad/S)jF}58[4dp̖IH)^iE4Ll'iSYwN U=u< CC|htxJï|b>q"t1s@>膴8@/a^: Ж69!hSg翄QFm[ !@-{ yEX 'jUU}%k DOҟԋX5Ǹ@;ԋJt eB8641_׼}u3xƾ\2aԫg<֕>v RYw蔣3$3T[,(Zh6US;i2H/yiW(귳":2%O12M3sb{8a U7h1;\yI"-z=9b$CRћjhEok/Y1ւ\adQ&S,ݣq}0lU(g\W+;|̰<#@hP-f UR'*ݠo [abӊϬ3JsԻ}悕h;(/7/VYU'%ZEy$4#CI A)<'Tcqfrj)\vmusV0> MHLvSIˆɚ~ &{D30_zaF{㢴J䠊/JBU(!K'su.gA4KJ&:ڜlZ<]؊nXh8mwjF,m=PŎe\|TbʹMh5灢'F>&_sb ,(_|=^(3 dv:;4A+GҥKo]Og+7nl[5Vn藊{NТG8y9JmJb_|^dN*4O^gpY.sAVosL3/+S79sܹC'G+o~5-Ҍ|M.ퟻQ4i;OG=Mnz GZ!8_{<0௪-~ OEV#@(i"C8ژuDvpXh]?(LAt:dj5*D<=V[ ]A-{ڧe%~ RWkZvjgkr/CF" `~_<˗ՙ}^^[* Sˣ&ƉYn@f iǖdRO30-f؉/ge͍9͚W^95>v`$祾= L #kuT8x)񁍒 &WgT)ܷY,$\k*Eާo8 CkNPg!<%!k,HwU1^R{ixbɞ-2;yr,~"2F0OLk:}md<ɹCi_g_hm\H ~]ߕ•n$ĕZdF:7PYQշo*Utr4gKMǽuH-"5 ǥ@m!͊BC{o#}`&^,G28lF,/tb7~o0ҧ0W\lw(bMB,o\|ٓe6ʍ%{IoUs-ܛARH <097Wm @vPj@X.P{dqgpkW {g0m>'5[qsV >kYtCqTrBcޅK$V-ټ>oY7iA[F/ٸ\L:ٔuF:DB>.&3? %B:,Xq/]Sh#jt}\>XYoְ; !z8FXyh~p MJZ<1S; M֯iru 4Ux`>es LZE7l}1fEѬ"Z`K׭8XH"!e]~1a5e]U=JҲUjhtܰK.~lʃ(F1$SN]ƫO˺AEӴ<&6˾ausUF7UXĄTKxRalKRsN1kU R$EMlv[Vtۧq` 8Tnss#[s7_2-[[)g![ C~$tAe;muH'$AA `敬>8z5X?ܫAdzu/>0N%A07N\P$j#_oS9lmug )$MۀMbLՁ9& !ad,Nf[KրrujG. r:]9~oٹjIr 2U$Jq3;Pz{IXG+Uh8E"]b払pjOwQ]4C< G|Dj󖪃VIޘ^82!s1 ˾gI5"40Gy99Y:Hj쳘eDbNq0$uQtb0nZm!D :C?Hf=D~GJ:X^Ib>(WW8%[+ czw5F/d,"T.~EE#GA(_ p:/> .:81{ |*FGaf7=M$?Vγ<%O3v6N))cM5ByWc< i;{۟,GSya56iܱ[*(bЏ'`"~.7uS]=p4cJ>Z].nuA#J{"{M_|U0%ŨƊCqg\\ċ,N)nx9X홃QQAG.]P#t]N\oiSBgD#3E5GFix8WЎy1ǕͼB] ld*vˠssA?^KT= ,xbsy\+?1։F;6vhnŧ"pQVT"IULa2  ƅ/ԩ37uzh.皤%|qw|{Եh[Eؓ}Xc?qf01J>D!l\ ֳm1  5`{Vb{zl8$c-sBrJ**W,HH.b);[ >_{&{ݟo n<|-D)*m1+ H-kú| a䋇SLnB+q||<1vARù\Q(?Rߧ G.?jt4utP%NZ@:iCB-a/dڔ`gr0mᗟt]*iچ-5E?!>8TQ !xlF l4{aNإUެ|] h\_lv3r7ӂK*bHg$i<!s1+^?aßT[:bc=PhlΥmH"aXw,:N"۟]7f [EDHdžc ?>X*.CX6q,a'!<9bm!(y#ʷP4wƲț?KTo|8՛lۂb{OJc%ܳ;᝞JBp 4ӼÎAaly$^zS&|AZnޡh䕼+ 4Or0Mtޑ<(ʏk8+p1IR NS'I$X|SN^$ 4I&{R"8?b =';zR|E9m.l F=mY+yJ m"-園9hhxBpgL]YJ_ⴸbӇ/0%T:YSm0mqQnE}5rB7}Ax!rS4Hy"g؊d+f#y_J`Y)vEҪ(Ml݆C5;"ZϼΑ![!e^mlBpD<8ͪNoϡ]\xasiDT WCcT.ӌ`IV'gg0} 8?L =c!|dꯒPeI{ʟNƘPiey0%&UZ~ж GWk|}fSY֎J>\x/gcX3Ee}:vedss?JI*UU@Ywjf-f+fM;&Haq՞#G~j]A~S6TN!uQ_RD>jYu]'iy+[D;^嗠>ŏ55<7ۆ؈m؟#5 4'#jD$:XY)4 $4|3>'hYͬ;460VM Q CtE\ K0%#C !0˃܋6 t]ԣ?Az Tk`Ot2f:kBJrxmxcNL) ow^KbΔ :t̹ɥo?"8T 5V8&}!7'.#GnQr|7/6~BT^ϝ=~Wџ̈́36 &6=a7u1]eC@*v $>MTH^ЙwLwl ,pdA;4^=@1끀hA51Ɔ+F!.jCsjXͻmJhڈpB4F%JPk96!2.i1B>ƚЙnYNND=j,4vHdZ - UA@ ۉL c iʯIs1rmEOJ2=E|rf{rʘO{s|8t{-EUGdd|+~~Um-_#eQ<#gdLܰ d(Z^HA~br| 㡝7e-h F^c=/ >IDȇM0ϠV_7J.;fnD,fB݄G kRkeW[bH\'>9wNT8 Wg/^Cӫ~#>7ҁ(>zy;;V}d8]`6W VUvFl:}|w:Hok%F{z23-^d.~XY%ڏeep49ܣ/@͍BGhY1Qg+F\Hꘛ9ͶGXʎ>~U:%KYCY~Ty ՝9P > DPY̔UK:❞5)b,wcq0r-h檙Ow#2~ρ >4xtݎLnT3]ʁpE\lIz[gEf$uY#dG "~ke~J09+V:v  6\"CEBVF'FrKYu$„:8c/}vpJr*#^ѣC.r59]T#p6s{R( vC{Y粽p$~M+2?4K GMQ3u :dMo}3 =dbawLVzxl:,c?7!Z9}ZW X.3q/U! -Ôu}d2]^3fhSt(~LPlG (?N y`Z]l+s|X1nɥǰr_{ eƛGۤIxmhLg]SckWdZ<$H*Pgi-q,78dkᕇ%`exŘ$n OY5AHcڃek|3LÅv~ 3[P@2+͵:6KX;7+0 )/OcvpSˬP9wo˥@oS|F^=f^;tv28+ D;ɧ,` 9rN_:8uܟ#N)׻0,&'3r]LͲ%e鍚RČIJ9+@"8nܸ'$mW1HVJhc'HG^UH Sq1crr, ֐=( [-3+EcQu! 16LV0o 4{(RPuu.`NѣfB̠/"J lVym5Yy"Ke(u˶^˜S]s&(H}]@ZGU8 4N~(D7 ~5F7Ƥ[i\t@>7F1b'%gHǚNelsYzʵ!s^i+~;9kח\zQߨo\E㜧r{l G;U)i:SǑ],Obry (0; h`~[Ĭ.ZPb y?)W<Ț7=A;BXaK/3DMR9Q4#,ќb:ϐf{5<`oB n[MO_?9>1􏺧 ^,  zm0aW!_;j9b9qm)d zytp)DgOwo$k?p ;1pSf*C M\:u#3*|){[\0ԎހH:"2DaJ^h +JYoJ鑄qȋۃ.|EenPQD#)|)wM]ߴMع#Hba1%5 03(FCNK1gsF4ul'Խc}X'pxWn_6ΗO_'6f1DlCo" z~@IyjcI眱#\[!-Nk満Us;-;L˶*> endobj 441 0 obj <>stream xڭteX]6-)- 0tt73 !"ҍR" !HKw% t|qϯ}~\q::{{$4Oֿ8@(0 H**vI/  {ǒ!PD*@cKw*{"$_&5zBua`_@ra{)/ DmC3V _NJ AЎ.'{(vFp Z#@HTRo1#Хdpߙc[BHݿ|XbCCI Hc/8@VB?$o[Xb?[Q;"@maW_aGV???K{I\_f+rU,;DÑ_eK.*5rW >kmWyw>H] )1ݟ^$1Y[7) b ADwy31:Ν#ξUq$<R9>v/岒‡8;zn=dQdsٳN/X70)6VeI kXy*1 K97Hs8NVqEHw+=.;x^Kd'veso^z81]4Z?]dDX* Qdكp@WUEM4f^V$t=PN "V)uÞI"anOU$/'<1KR]]3!(hDkEWOš;GV91?3+pfJHES4$x %P>`!&bkq}o*&wYboޏ|<9gEolf{}] SYG]mBxCe3 x;LjGV'oAX?" >uD7jWI0W(im֘fUVcVV#`u9K?-Hmrx?!î6ҥ;9n%T|e-;qÜzQ0TME']: didV-1!HgY/£Z`-Ie lu ..[D40BQ6~Dkn@*nsz]y; Dij9.<b}CV<1J؉vJR'=e 9$pS,FI#êl Wm.h{6͋]eE;Pӳh< Q"N2-qNv2PLn g`π|8WbS/0ޤJA*wrJE>E((_cYym"  dsMr&Y/ME st#>>Ot`gyz#jCLj!1O^0n^f:B: \s9ykH}dstYRT^m9PS}veijl&7{;='p*TUO]xbG_Of "Ӯ &L*q5({&|b!} =jf5~+`@%na6N+|ez߀L&%)~I3Is'JY[&yt/ [2[1jQ"7/D4Ff4t˳zo6~Ytzҟ494^E9I$>oNiŖ ܇K>eKf8mU ܇Ց P_N|,wH6,>m@q>{T f"ÑO ~h Bא/_sjpwZe\V˗^GFU+i!>b  "Z蝸i修pa7tM]:k@WZ] :*ĪԣR6e,}\oTf @l?oi,?P'UhtI[ޑF[I],LJԆo.M>EDEY֏ l̩ղn&3j Dֽn lY rNRDwтUe|g2BvT'W 9x5ݙ%i%u!ڦ!t~8Ʈ `>rid 7=VBCb/"gbبܐp0OQ k [kCGs4+,EfD"Y WZD,M[hu-}J}?;Nǝl2?k: dSeMҟg\~\Kvث#9Zti!g_Nɨ0~pYңIpA$ԾT'f>1ˆdh|$pګ.Aռdq3V}mf m_ݟ@md=ŗ|j h\&M[i[)^l Ѫ#QAl5~щC8nJMr6^=H'+?W-n*MloHދ|NCaP_A3Jaf4$`@aMŒfmřjZ:u{8|$~p>2zx"*Qw!^;-ݰ>(6AK75H4X-j=4@ _#@յĶJ>4͂WBE,cik9' UBߣLq~|r}qf5孇;m:[EHc)ɫx:(,Ԅ=-֪X$`!tu)wWTvгkjdvOjC缓z?7M:ąu'WK2BE:A 褂xnK->{tS^O ?G{I=ySWX{8W)ح+hV3r#f^&9^M`7⛇bҗӕn»g6A_qB/]پO?MzW*ա^ 3'#%w+?-Z XpJ-c+^!20=5Y 0me.uRզҧ\~׌\I<1n:nZ6C"C '<$|cSqb;+"%߶¨jy3p%]E  %vƘavz!/j5NG#d )Qx_S2[{E!6w?e%ΛOG0%Pf2xtnp1K0/s=9]|gDϔEG;US3+ sGB]qU]ǵ$:0*s{;8 9׆ m mKs^qqV۾HM`QMT&YvF 2-adԌ[s]ϟL&?j^˅p}_Zz9{dK)an6hEt?`l0݋!,-yþM*D$d{#7 b5Av#p#!#NaxH)gkGWM;nyM듞u3\{*,y9z :Ǟ ;"*]ϩH$d~Eh_ ÅBQ1S&SB'-p2)KkJR pg'IAL;2+MfLJ?reHs~Io~P׽ij~m8QNY]N;Cl5*ߌ{#kJ=:!"6ѽGʡDSlq;&3A`Е?@Ƽg(y_\I䔵e1?vtָn۲W.*D٘O^n9zv!ѺC)K֡8Cj3zg2.]WtRFq]q- Ћ}˫%>D/MXF1v9wēc&c95g+d-g< Zg endstream endobj 40 0 obj <> endobj 442 0 obj <>stream xuVuX۾%d衇A`K@)E@Kig߳ϹǷ~1#lJ8jl=Q6p ^='!dc3]AH +ؠoy'O (Bb`1!,W )A~@(to)twwA(':VYw_$ srs@DEE1@( .ߙn%p(h߱:60v '4]zP4?mp{yo)Pۦ|978 nGKp'TUo!¿1G((P ΉwJ_w$7lwGl\Q@E菲HOh%}Gavh-V5mH CϿVjekꪫqE!n%y!yDnr(*$OEV*ۉUu!a9@@&1 oqW_-Jqm(w6q 4GGm ;kL0jC9i?qC_Tf!?8' Bݺ GJEwogD"oݞk- N!##Z*eiyW? v x2>$Lp\}w,3$Un$V1ᑷ\+y$J 6_>htMx!ۧTrb&%?LALfKP@{}SEOm ]16sX3"s5SSV2S=rw'(liM=knpjUiy˱v?D[?$1. -~ pçu O,2+ ޲f]{S<8$08:p8t@si4F0n1KM}ieʜC.Q% Pշ|BX9ln59,<9+q3~mDNI4U(B#ew<-Հu2j16e{Bw[E8BO"'ftE?w̑fzE̟$nZz 1 *.c ?/8Cr㣱_shaW* v-1NgK! ኉ ]I~EV)hvWd\RkH ۈ*kgͩT;q~(\ީXU1hUK:04EJu$IGrD<_S{Q4ι1EĄk11RGkòr&br*y( /ELS`,ʚEl_Zo?џκM kq|Qu"i}j pVAt~q$;3=nҊ3:IwSr֤Fݱc#qVɮd`v^ШYaIxIeUcYm:}wk D \A>cͥ#NVâ_?eMz 눽!29 <#]_Na$?kj޵9dȢr84kq]25 l9c^91.B'1|`UhA0P$ cG$7eᅬTb2&]'}%MAѽlt% ;oa/IZd'6-CIw7q$]ɢ%t|sEu-.$"H꿐zz.ͺ._&n,dI4+zKZ ~IЙR=z[W|X*$DāoQN-Y\솈 {` MA J*yWKdz`sYiaɯF75g*Rnx. TuOm%ns >3 %g5 ϒk_Ƥg+M( Cao~ 7(HI@p/p E(C?xhe+}@FlD*_6^un-1r o-P9̝}fҎ@މez uTdaή[plz]y_EW2;Oŗd_j|R 0Yq}XyX'J[G>?>Nr\;?¸$TLhMδěk>A|Xo"#bbDDHbck-j_=W/ČSopi i-Sm7xGѮayjwRijl\k[n/[t^}p.0&bP3#?)t)k}mg PzN.TEl!~4qI1;cG何 O/={ۉ8G3~w VWi?Ta&$:4r&cXYyZ rqp! Ѣ2ʜVt<<ܔhLȝ4NvOÏ :D<*ɄuU䥀[d`(ĩ^v\pYe}Lvv "a;HVJ z1X^c/`0#*re`Uغb/XC\T7L̀3;بJ("^ q݀Em~I6r= "v:ŭhšk*+_߾9\d.knȢr}44Y5(w5x`(CWv閬 DH; nJW,φx_?bWQxRaJz.tzEogN,&5`' >?H-/NnXQwڑYb·yvϒz=ٔC}=7{ 3pR"EΡ:Qe_}0_>0gkۯy}#%Wa9tD, $c;@ ̲F 0Jw[uLjw CgnЩaOk҄՗ʣo)^yQFex|XWg ˬ,E.T̘{rWxpleFu_rIk}:6qVmZ11a~0w¨mm~Ub9\9:tDR {>*YtN1=l';؎AoJt5gf3_ SRs ;/grɂBykrkW3ӝ{x_ *dP? &70:Vu#M\hЉQ7Z|Vwه`}kU)n|w%Lf ʫ#\`[9I),=eI**V XS^ y8FF4x'O蓾ӾW&!ѯy$]dѾזlV_Ѧ^ɿG]-Z oFPg_ 4.RȀ)_U>"9*v/J;zq@H2V9ZJPX׃SZcռ_N{y iЊ-wy{Q9^'I"C_Fu+ksY8oW &8ŒfwPw~pwkL=:g0 ЌugdWMWASfvUVU<Gor_[v)PqT9VIpay|kc\o7{lwkIOvjOSr ݷy!7$To4k P K]+ ; -3܇ C}xGf؀+Lm:\$ɥxgI+ 8ø0"G Hs%⿾ Ї)XL̙g3olK#/=^nj/}&[6ؐbq`#b?34#hKFJ=b)|R\ fP0R1&rL-66,]/۬pI g PBXA0%^:^ϱ6l@ӅQj!]6"=,)4gEvO Nw ɶ6mXdM!6rX?62*JI(zT۾5ꆥh=I(967Ċp7Squ<:5r}-HJ0KS5䱓^1B( % 7SˌԀl뱟QWI< ~)^zH '&0lnUBZ.]^\=ק7yʗdkڟh=|F3%Fy}>rh9 ?F`}O *lE^hh0*`N˯}`iB2z^~,GT>-ۊr)淙kҮ Ǽ#i?"+jQ_yu֬Ypl3 Q=N ,AӼ9yDsd=g Ԣfߏ7?~~NiDk ~:AFUxStn?H;(NJL~>IaZ?Q҅1~%xOFzmy y^GlrdZ dI m梥%,G1tG,k{?|̔c@ y >d-Aw$ [7M6Wy8KcC ro &|V*2yˌt.z{Qt@:"3H6 EQk>oTgIBT`=Tr StYn.'yݶs-jW;&۬c}}(K:/S ݵܒLH5GE 1Nea>Y93ϐC \D wl3!P_`'fi>2oWu9&> endobj 443 0 obj <>stream xڭeT]&'8q]h%; w0<;gάg]zU]nrbEez![#=37@dmZޖK^h|EN [QC' 7@hXX\\\[;wJ:5--J1df|pZYm>C_;*'s d((jJK$U@@ d mS[տc[?92|rƠO71` rt|f6N=plM)7 ;O Og0E[G'Gc3q::ؚ~Z;Sҿta>N G\F@ 3g0;п`8;lhf`btt gzC;;+ya99LY>s;}633+R6fMCtWO&6V)Sli)"\M8ʼV]\gflՖ)v9h@mu9Mi0]JŨuz]ݽR 2h++oO@Uh:WZW.^iV7 Xj_]-hWqWOJ|:E?)nåldkJi~\LxUhGop;ɫb$=Yr`P {qvh_qMb%R̬Ǽ% )wwWNx{w*QGݓf>;I_ﯙ&5rmy%Q80w-X2u)s1|4@xy 7]rN䵚R [OWʪ;kJ`nBe~Q&ȔD8P~ܤtѷa;vΕxv+j} A4CCނ To-/*9JBsi ]cCDžXM;?=R2Q't-]5W|rlO`)pd:`dL~q彯u(OZB=? ٝ"R^c=v,.qpз4w W?l^'kHR.Xv[:~0kB,/rRgе;P/}Eb鋨S7LG&mj~Ȣ Iݜ dՍ.}¼-؆=omFl>*j'tQsuv U%w.bdJB=4u+DsDZ~؋-/f0D皪ɾm)/|2|}~N2hm!%UװNNԑ/I!U=RV%Zql Zf+,U|²S$d OVq7wV]@" sr*$!mr{gQZ [|<0ZۡFΑ;| ߤb&h[/yLPhmI" u!Pc%FoIe SF0X}:P^qf'? @є2@cҬ^Qo ~W_m fT)$Ic( OӵxxޅfGl3PFSKk>[Q=O}aCN@£gDunѽii0$rM\@ͺߙm-{`f䶔|@q=9A?XSN%zv^iB+t.9]4v_Rfm?w;>[Pu\7V:A(o>6*Tg^XEʏBӌ }T>=q>XĖ! 7N]%9,;9^-,b%W=(fG<T/nsҩjYkP\wcIGy0Bhq} Oފ}Lς,oXqtN+L AO,j=&j+8\; S' i9WEi~A  4(>qdd95?fP:= b%IV!IUq r!yq [ 5"fGс#׿_E, \<pۏ :T9X`O_-NKvZ{>?|뺳`YUTW$jd?h,I&ZJl~mCsiB(Wʎ%ۊhLEkp- H/$@PvDGuR$CvWRf} NpXD+&wkT^dٽoϠmz 1l gbzÖ/~Q;fDFQЧT2xq7:ѝ7n?|eI58BX"~cr<;fF8YrɧOٓz`̚Jmn.hdհ+yĚzFQ*~ԛCl|r-(8 Ncrc׳œ. ! 8Rq36f̤FKh%zjo3iњر"u*װ1b nwX,PKm9?)6Pi t$ r.l)˾EveդD/":YemuFᵘrWYڋG:<_}gaM[=Wwv[YARlz^z'olȶl"$gB^w]  ;UX;f#ٯmԨɾfŒk\]xœ2.fJb]aχ'~ͩmZBa_'/:JVl",D\Nvj!6(y`'C~| h@vX@Y9p oMD~ޟ-=L⡀_)B3_樼?zK rY 0\J,՛;9-brX$)0 EM<̼۫C[6,j]gz|CC=󗩢`(khv:sDڔS*v6܌V%Wע;Z+cǷq[Ft3;L"IȜ/.+Ř+-)ZC``WfGo4?o[e*<:{!/"nw0Oٓ7+u6f8Y[eQP} Y;͝^? |J5`*'b٧ʆ+}3#6."﷔4UgZ >*u.KSFQӽp9h|)pY'0F$v,Z}Os^ 㜏3YUL2rSx6op2+!k/|?-e+('g"l&#k:$q|ѻm7~\88VF{WPݰ̚-=VkcD8OInXN8=`WBܢ-¦!.6@Uhw `5o͍IAc8R -}몮cFۛƗ/öP:Iroh¸ԯ` G5X8^_Ͷ4rJ;>h˅](3~5J>Z<7\M&3; v:aI*WLoj*P"s+Gq/G,oWʊQv-z"Opzr|"W61WT;# 38S6Z'E0J w ;5|f Y "%, E4e Xzas&Ka akL̏ZJvd#o_M".EfX$n0(M@n?t֢:TtyPVcvbKU[RĮUczlY. dodI˚Y&݆pHPLdlu:n;tr CH(Ce1*mw TSW_|Mm!\h6~M߶+?],:EʨE~R>:zv p^.; ]W7B++u+j>v $ϑ˴tεn+u6krڑ:s"s{CM|" 9܍u&+M8z=i;nF~9^ٱev+gKҿ>V/_lH;ŞZm}.0vězh@oFvID%yc#ĵ@W<.fUIm9B oqKi%q5:,k Ɓ:^2 M*s/33>w#F]MW Kr}@o1_ ImMQ)w-":\~.$S 9闟 <棻' ^}g` A*d{D@A=_(=b]cu 6b`[Yd#)ҋ'(ViP qܪktxe˒JN3_{^/ U9iKw܎it ѝ96 87Әni˽;X٤[?Nw\o|. .`>Cq.[˨^l~нccqU*|9 |ƖX,D]0 qOm%'oLzFr1]G,W"fFzsxRMW^pH[;:-OKH2P`E5Z!cL1^OYi3 43Ul'/iYr𩕍6 l/.MNrsEӀ1IڏD{e\#fDƷۈZ~ gmM?yiFCPt+jrRYW'*T Bj):0&]|l:MCrSZL,v+Rg6fcHk 4f>E}UW‡{TM7O0A<:v׽$ʤHA^WI{gP1o<- Oy FƓt?}:"j]B̷iɹV'B,nsS%v"dOXϑ]]}0[y(i/iQG~[2!>n6ɛ PnٻL@iWgm1i4`:kl0\rb6%1ᐈ(OcY 0.MlDƨ;ԞNbU ^19Vy-Y&6l QDʫ44}6yYh5ΕD)#Xpv';-90o(tPGܲGzŭ,M;DSH 95'd|pEFeb&yA]~s1iK [ M\kezӧcDWk(HϨEvuŵ^ڴkW5MaZϦyodMed܌eșW+Bh-s۟u%n-DB&>=sl .g^A_Il-ZյN-eDhdpaꊭcVEU|CGtumg%Z N"Hs60LR״PɭR)P7w#D;\ըɛ9/3U8:u_/N.0;xq@ӡI YY哶ofͺ2es?XIdu<,akzwov*â2&Kܔ9eиNՌ ֋ۻ'TF9|K1fSH؁* [Jc`Nf) mfmx'zWq(\ }#FۣV\~A Fgo-hy;/^Ib3^ 2Mc"~_w` br!w~7[7@U#Sg_&ACV\O+j$4cxz;bU2o(~/w#i-1d]lH#BklX8C]M´%jϞ^Mh9`0nU?z-u\dQu,/gM[zfŒꡑIFX狨 eT`t-*/iFvVKt@[.z]њ鹩u(ɹMFLX[:W(gBL #]kB EyBr3h{!KVT%q99qxkhXZ)w{־gվ{/mQ֜_8c1\{dD!%4M^5 wY6;H*Q&g 2ԡCCpf|{@y]";&B3I&_7_:zI=Ȯjq< eMR,{!Dg/eeW4=![b: d̎:w.$AmU:9ӻcʨb;* Ԩ/_ms ez>+jOC / W]FA*#7^.^62ceY6 V qi}$Trxltr1J~0aC+!n93c<<=;C@)Pf#%^LuIRGAD9Sk6LLQUCv b WdP>"?36Ά'uqBȝ&ǂY^="B;"ۛO# 2*qu3`wz `Ix+7/\ۗpSG*=*8Mل dzF8|Fμb҄ Z隖iva="{Q*/!)1'Xħ^]+"{Sl4_Da+Q騪>/qSў\W!igOZCJ6AZ-^Tc_AXk{dT}*;2,#d^@RN+24jRqam:mqm"(<?ꚰ'Qd@N$oef?J:GfkpVDO{e[+W6'y.]~tmOa#P>-,6W;b!!{m͖?IB4xadooKwU A?N> #f㜔nv?LTS=&IV X%g+!/ dV-45d ?VD]+I#WbAַ+AE"iN&A?]`c]# RcceYGV9sBܿ7.3'o̡ǃڄN=O|}3&A׸oI(X$bzm"Y7QcTk.5N@b2uL.`$ )]`ryOc>y̽Y_icRF4kJHZHC趩tRBAcf½TdBj*DL+Q_q$yRDmVm!Z:i5T?hB& (V!XG\ %BfwlM4!=FwRQ/DDAW oXbg;tFŝ} yA4`ޡq) xMrFBI^:ІL0zU 1B ɑ7琾$ AF7dd+fa}P x ӆx$3>ӖOseϬ^H̢LñvWeKluee9Jd7IFBuǁs#y؅z |*>_&}Y>dC(yjp9[k"1e߇- '$فpC?fkTWmk~?: e)L{Lzk-tp m`̄SŠGϊhIB.HEQY.pSSx'Q!\A!bG8|݇a-,D9Pb}eV,kljuw^LaƗr*@KNXǜؓ,~TCyOIQlDuY'V8];T:BСГ};"3,ҋ /^i*s̘@ /EJAs6 GnDyF'dC4.S6wlpl+wZczr&n5(:џW4$Wq7&wEj!܆unGXok&8|ɵ mDSؚAq5/D1 a/3\ͧRD'\#5Æ7> ZWQFpf[PYn{: G[4 };Ug8${CemY.F Ho&$5:Ueՠ+e; D6bܼFIl0 cHFE5+zweaGCR Ԧy**pͤpCY h'AW«ש:nϴ߯kU@^:Tx'uE[lk~QЌwE{m-kЎ4{vn;ZQ֐ m8Ceu>ԛ$H^UhC';rMtS gX~!VpC5רKA-XY7R6?!5>jUZu'.U Lgw_M`/ ~rY\zz"UB_ mi% CB-}a?)|?-EmG2c#3Fugu+FH0avEÎUS5aLt.y7H-/$ `6J!: *g {#;`G1$u~}㣱_;dyƠ7QwvX2:;:] CIK*3 n';%jY?]qiġ+Uؒ `^~7p!_D3wȅnO'Iv/B2нP@G&m+y#y2ԠБEH L)YѮz77C#ϒ |S(6s.E杽#h1c* gyS"Aޤ8745[ms|qf;_r$c@I:]'pX|ۥ^\po{*zEu_IKC%v ( 6v1Iff⠖cX=G2Z:va~7#Ox uι@_e[2ʸuw $Zn]č LU"wX.6|4Qܵ<46G"!U-+K

㎜gţD{_Zql:sozaTh:4x^lm 7FOo$i^klN5`VE~H~J"aU Iڙk!~fߚb+cx]xšGgx*oŒj4a[ҕ'*1mRy9 jbY+VցO°́QzU5c"{`_ #D69EuZ ƟJbo@ZiMťt:ml5L28g&:)f&M)~|%{k?ِx+;`LTN> endstream endobj 51 0 obj <> endobj 444 0 obj <>stream xڬctf]&vضm۶mbۨFJz޷O}~1&؛DYA( P7uwUuWtgPZ9((\&n֎&n@^ 4Xxxx(bN.֖Vnj U-::c0_OWkK=/p,1%eE)@ t1(Ỳ@ ?2q\f݀^f@T'w+ofv$Wn\ZSvtus5svr,.<ݬLjW pkihOIu3vpe [:ٙx_iZ;Xg/_uM/+΂oL3- G NA $L@ 8&EG!w,3@ -_9_.}Вvv&;wɘ84ExRlN]ְgUrsY76w'gkTY)'&rnD&37Jf{ RpcЏxnD0} uYM c ,?=cul"h/㞡T2vZDglj6 zQ<~QFJطP<=veIp . ϝʕxH]T &(^AZ6mmm׋=߯PԒ=dUjj;aVO>#ث&_Un,2Њ믓2' n.G4 jrDy u`y*VfQssm]إu]?lr3-K5uj?Hqrf?MHM*\@v?*)|\iq|fijmJ9fjRX5X8o%yJ]΀alm_7瀃![eoK 05p?`G֫(iYj /4=u:, ]AŒSk IA^Ǥkz"F?%(U$Y̷|PĞGLBg#H wHFMVR^InAG9f/ЗHeWFC+YA-߱^"FC((z ,^Gc?Ev4pKͰn*ھ+P J ~=3\;1; %70ؑ=s#5D-XI#F4Lb g J%QUMceZiIt;01wWyЗ{Fuc&N|:#LGit""u$7;1? no-އ#n$YTo[1be;=Z> b>kBh &x 8ΟA+N0dGW[f"5ihu_ٯbc-vsVsrYhv~&{#'!Y-4d+r7)#B/$]m6:X얞CZf*1!l [13)㎀>Yюh;K'0CNQ -L/9V}K vvdma ]cIi‚\6Je -\<τS|hy|\q-o)4ϻc$D(Hf4_Q]7Eh˜ B9؁RNn#H5S)иo7$V`#U PN{"Ş0lc,E/ @T ބSQő DE?y[O7 B(>RJwEDIY2(EJ u5?Mp^:,)H}wߌ[g\ &\'[?3P};59׮Bﶭ@6&J.%P<']]=LDFQAHtC)ͩ][ V|Vx*{`AxXIkl"Ő,07rT1ȷ@όaj/dVp.,7bv MlQĞjk+YQeqF(o sQ/@o)X^MSuR 7-'/G uZR}c̎.D΄to񽽍OzlS%`8TzerN_㏣ Y^Pw8,*A?o Qj y~2>\&uJmc|qD0tHf٪= 7N>S6&[lC-{GEu5H/ؓ`!?h|@|*li⒃Z:TV)+i͗^/"Yܣe);.1gXwT:905ǂZtho MB݂yeᔆu=BBuuw帶ԭ>q v*ISB\ SN`f`qDxd[9O< ?[t1 1Dإ[-UR97*5\FޏaF!a׬A;Ԕ_#g 7(P`VigrʣaYw=}ɃGm(i`efۭ*<ͥOb@DvsKDܼ@4io0/#7C ӗӏ'vD/$]ϴ$y.CJ e}&7$TlK&v zY (xհF- "?'!\($[ ?*;v3E%3yq` r;.^nl{1#HPk W\kJ2]BV !P^aE/7.JAFb?h\_} f=7BZM,J7:+3KW󣍔5n^yݏ!d j#Cf`Ps8}?ve hɴ}%`<. Cfo˒}+oJR 62_ĔC9^#; T{T_@8ߡ:?iα֤AT365v[6qEk+ekR dNc8RDV>l60> 0u>ʑ/t/`'{S\(Q:K[eppS\Ug; _ss9;[%+mzlW_WQf4/٩4f[)_ܓ㺅gh^P!ȵ7ŝF*g֣)bAn6 vVAo'#6I&G5HK'D[€քZumQZך\+N^K\p9=ڭ*Fyqs(kWzcrJ 3#nU@E w0\'yNQ3:.}&=-ikx+ȥa:.ҍZml09Zũ'9A(`bm3אJst<=W<%%X)dBYr쨯|Y6h xIcv6N-r^# K^ٰ |eKZ]CQsW :nϻu[ v)dإf61K6!\N0i_*0{zħd#gXwOdpQCVXN8!Ww9μn-(_yԺs5WQsrt8묋LQd\cIN(_Q z $6 MJWuv,yޕ쎢$M5}cvdvj\-iLMawWNmדMedz84D3|1RYDlC6MUgd=|'*IA$6/JJט!؃\ {UHwI`h0V =, =KF5ӨZ@-AcOMI} ,I3旈b(O\9V7Q{h,K%CL4"{t5RhaU,Ϫ,_)q[UV`8c6r[24_hXMG%\/]"'qTETƈKt2Z?v, ;2.^}1YnU3w^5"{. ztkLG;'ήyXȳ+3HX) tM<؈GFng7~`p7V`˫%r[9_icSؚ"دWVzhߏPOIuM9>~OA4K}2fx%T 3ᾩٶħ?Ot{d': M2zXL&\.590C7z) Vqa ˥ >v cnV,#|!g䏽795hR vͰO>ڰ4X,w>21W~7\4<#d*F#E66^[}&k6dg55.MrM>ojGƏz=gg?ewiAK]/-׌TE%[KQȗFtHU(Wƺ{սt^rES)XаY@_9EՍbSߖUⵓ ru;`5hRg6SS!ϻt 1l<ҷ7o"ag>da\nrd` NQix*8LxNLTVA8?ĶC$,f oT%?g VϬW25ЪUL-$ =pBx8QhK$ 2*yOz}tr3 C5=&?pauNIղ\uKjnXHw~d$ :1T-n +j-`C';3zP'4ZmyXnG#0HħM|L~3frr :l&D9(F }ggp Y`9wƙ)\LpO4y*yUm$} W31a:96H7ԱFC-RE$q?C>T)F,+|  G 3Q$˪`GgWfJ*w_p~s-HbãaΝAgWj]met#P0[ɖZOhlr{7{4{s.6@KOi; XU /靖F FYwo fN JdAT7*D1,2#Kо#afd4?P>iB5|S\EIC6+n`1:w)?_++uAUnLp'fNsdJbzl(\dcgzGFWVau<()ظŹY<k+AV% v>`y1P|^U;l m\"- (JUX>jEwA2qRiqduTuY!Tw,S>M6?5bEA{TJ%>6:kܡGIkDK7X@v&n?qj7| y)$1ؒ;XSE),%lRrt;6thAU|3PJ 6izei"#'d(l$׶o.UڊV1r0Z4W}S/ͱ}į2&TQwm9F Qm (¬m5=R3Y~b~ý__l/޷y1WgTfzn\f %=o9PwCɅ7ܵX*]%n .zYsH~ٻwf~DO!}0)~įk-*m·mQ"r`&@ 1# J'={ c5t!e0,i/j. ??#Y4%"6ᑥ\(/%O PVEM#~H|M\%T\hB""ĉre8$0̗0oa~DM ,Ӫ&xhp+W< VF^QJpLU2:?p:|tN- S5tDu#)x*5y5WT*qئb95&Ex=g 2s|Q:˟$"7T6st󙍽-H k' M =2O>*sN\:ubzbe69zĤF#Ek]2Ɍ#?iwKilp]Gd$ n';[ "gק׸84¦ >WJg{'l|HŏZ~[򷍌n%QD=Z ti],Y^oԑ`D"gBSP=kX'qX?A&n7׿>MN=RV⢳+$Mo񪞷`' Ǎ~uk 18t}KjPC^ʐJT9)WU…JPs%X<6QK~6ci-ڒ:AB-7K|ym.k9Tr3AO6dq%8 ?n0kOkDͽ+b5yD ۮJW*:anof<9Ǎ,O%7ڻt8֝.ѯ #0ޘfj5ȍ~5!v)G`a38l͸"̷y_6jF8z&A4)͊- S4e(㧟>=mx[h~L@SfE4ԏ!/g;E;wvL&6JySruLL`N p ;erG s>A@qzн>XL=4H!G޲5DLVM֝8Y5$kZ~ɴХo|8>cInU "90]bG3լj\k#&IM\g C^_+^FHi[XoF(m_xa3h[T ޵˹$iPߜK3:Q\[xJxAD[vRTUxTT^7Md0)bKh^Q1P4yɘk3zJc22LZsxN$3߻U uJ`؊Q1<>Uܾ;7a79‰!{q-h6G`>Y&iĥ-u[tIHji"ވCt45< 8+bܫ\/fΚ2!~HqYKsgGU3F{%آX[1͌RF%(ĭ/'/ϕV9yMħ珘3hǺqPBC2Fl!ve =fJҐ#4wʄ67c³xfDξP_2, a6Q3]MLHIڧtQ˞wa-6e#˰$EI!oZ?]?,-Ԗ @,c ]!D ;)\hHֆQqb܈0_![!R>r0onxоN;T} 4_QՒCcS\<{ǂW)\@\9s]>ezY1\QUHf|$"Ut_e\ִkd9mdYr3z4rp[T Ck#,Iq$U.UD e O[t Fˍ/Tᰥ6 殺Mb$H.S,c+q\<w zjIX#VCFje/m̮~F<9WsÚr e)QxbPWvxQ.ڬCZm_RMNPϚCb!uHE5\7 :=l4@ $'f)7GZ^ҫcaS>nO?>$eFP! 3@JؠyZc7WWX[593'!%e NIc?.*$RKkOΨ+0~)#R2EZ _qrv 6i?m *G< ^ZWiOXwڤTR: 5L]KnO@ѿڏKWjy2%-i[Ƨ22yvzGmr?rؓ+Ɉ5nR}r&>$E".YR&a6c1d1 H2 :'95y}jw 0@(h@w>*;)=տˣĠO[AEeTevR&R7L[ȶ#u5a#15N8D5Zk4ƨ"oPbϷX=bIN,&p猫hQٵEPiϞUGf 1,XrQwqF'Z !5Z x%056"Bq<4ք:=L= EyI{ !cWQ& Տ;^ D -k4^}}d! ߜ &ʏHg7*3FEL?ω;wlpCX^iR>PB>aMKܑ^-֥yo M1ҩSA[l1TsPX ]Jڏo{tIP7\xl71xxz1n֋ϋ#ΰ!Zdr> @x=|DP3'`z@tXq<tf/_kJ=IiɪU_*rSS)@wu@:em^+FL ]ldr(1Gaɟ:! >fg8mP!~t$PȨKߦ1j*=yH; k, -FMwXf5wE-5F躡~ѵܼ1!qݓb TqAlMGXJXoPrc 4-Q:P탴S陹$},JypOX؂ LUVÙ+d]>Ff86?Vcn wA@6os̟?wA^2X >n I\=DdbW'H,Im5dC[ss :<9LPOǾqH18b2ʞ[mxXVb|nUjî"o:Ummד.'C))ʹtN9_b|oYPagCI{q)*7崐q6-^?H<=hn "0 fƴv.hu]?e;'Y:`)_@;4X%[TrC|J*FOL4SŷyՋū\ڭ$-y7Ua8Xv4ǔ ј;sF\IFcD؃T(&M3:cc aO"r>Br w59ڡ38+rn]cW_ %o/cE|0\"ϴ ce5>G FtF%9KF@4ti\Oߖ6JH&W*:If;q^ %cф7L'I8`->Yj[NFx+NV,Pԡd"D2A@zK VCS\ȲS0VpIlCuT&B g xnzcEJ*DL*%_}՞x6bMl|ڒ[N8hVֱdAJO]KU Og_3!6$lon=0:^&;<gGť3aNi GnEʢOY EGo83ڰӬ1-sJL1i9 Kvn|_ 96>OvE\6yT^LŅ֍O iƇ͘ wr Q׬ W _ljYOQ;SC,l ۾rfU!J&;Q4ՙ54ӜQ3<D"]) lAo*b+o Tph ƺ'P?ȠfX[}alNS۵ +֡nyܤRo TX a<(N;ݛa&"t,줨`AFҺt."qTIt$B+ 6!vi6Iɔ;ER&+uyX9?1r0A~ 7ʄFoN֧Үd.U2Qw1ˡ8.f*AŲ'!57BfĂNNOlZ`Kt5 ugLty_[ɕS lUmkF%69X+zFo1D`] \`048UՖ-SwT*H)Cq9,l)ubR>JkEq0o)d IZK=Pٜ4&5!o)4]%>JWi$/Ȑ;jNg02h& Ġ:ݿ{[etͩHЬ}.!aIyՃ$&oZ1(Eoǣ-)~A݀zˑg*bgt[-on ;T1ܩн2ʜ|ch"-9 C8B'Bk,&OAAGE)_MJͩքF.a*Iz(, 5|tQ-49<]fv]pӲX#ʘ1 臚49a/@#MλsD5a<?m&Py1 ]c)T]%m8)@y;Li%zU!K .jv~]"Y\l 礎\QE^ZU.r C 1iWt%[4U,f8^f)9m4Rct؞x/p]sͪKv+[(A1ILR(.fb؏ڟ(^܀W;شKosO>| "wQz_Inp70j!]8%аE \Dm/Z~ȴ‚yng(pq_Ա1JWgz G:8r [WnK6K5@cK8+?xq: 3 > Mlqy@Z}(=t(pdNB0w⤤-RJe@BfX~5~΅;[ ud"m<[UKIŢ"|F7s. !j2VȁV NsۀzDL'gYf*iĬG{/Z_y0`.Ed!wzƜX߈8'USP("H1ӷ>8TuZc3S %|d<ϐnƯI@?AG-pYCtJnK ն'QjZ[ޙџ +:Qal5E bzrZyB3dɗUlsSl2lR t &A`:u&4Hg b ,)4 Q64{j~3״M$?IJ[~7 [u c9TIԠ2:s!G +t ttӰ QE1$55b;Z5h($8 u1!@#$u@!{L.ExL;[6Gt@8fƌ0U0WYgjP sA)qN~а.>AV* AjE:xd#(k<^ǁOW_E.V{{e%>ALRjUMƾ p3aQ\BAտ/qPM q;{Т*SF䵨Lx9.WRt ɻYѧN6vYm73R?`W/%pD[ `@0 d_4YSUoڡݤ5C>&^/W$\=/px"oHj0ЧdmXh*y9O]"ϙ ZW8~[g|''i-EE/՛5oXٙK-#S-\gd9̓N1dFhk˂h(6 I|HmG?6JZ~mITٺd+*9oc yj /`N1[kj}xA?%o$v@)BHH1y %viHŢLg /QE,pr[]D607`yHs:D7HDBG]Ί{@ytoGsXΙL|9aM gW%-w;GμUEU"GzLXO6$XIz5(M.KwVz@Czb{lk?.89}1|(HUm˸PahNbN}`H=Ō|AnLOYepoחEBoz(F7 ٵζv`B(A0ITEm88]UT+S&  3m\QO/- >pcXq]>_Ļ+`psM8GtFt] ) ˸ŬW )&/nx;&+׫\{p>#{E%ßň7sS?:M˯,- EE5v25O2jPe+Dhb!w|zܞo> endobj 445 0 obj <>stream xڭyeT.]B!4@i\www<.{9sf{ѽޒj U q Hb.Pۙ:C!, +Wy- At]{) HH$ leRa`bb/?7Kg= d qٻQ 5@ 5` $UTeZY=- UW3[9@lw1,!N{ _9q;g9 ar b8o3 hv.䖐rpiؽaodggs' ͫ?t os׿R{yC]`{g /_f  w`{E pY,lAo4oο[C. [KV7.o(l%?17DW0z,@(l7YYP^22@Ǟ-=m-[௅6Lv`[ߵu@?8 q{ pC v{,T.K-ײ9قAoE~ߌ ӴU @ {ng&&lۿUߺEO:JC_m^ qchy 8}^v 5u.'{*\;t_ԆXN&(£că;oC=}$L94B@#$OSF?On|pRSA$?>rw^MJ:7X7>dyQŘ^]` b&3}rm\Ny-Fϕq"qr[W_%DոpUk>#M^rΟB^۠Xl/MP0~ WMs 6s|!݉z%Qj{I姕/ |rS88>ƃU.[8k ̮{~Iw9Kfvw墦d s`%]7}X.ZJp$*jVH@})hV3IԂQ)(:B흯H;\< 'hܨb?|B< =IEߢ$u=f:?p߉FԔS.v_ӥ'D5 ?7gZz uGY5eMjaUS1φ쩩b%1Eyt_ə쪛${<ȿ+Q Fv !׾_oKZj\j=ZE|>iMY Rv,r=^g5@ ZOD1|WyXov]kt6@{4)eS. 襍Q`G0,DwK5; rYsb,?l ccBRι;9ecbTrƹ:"O!_]1'*?+ \~ej,<fن{j7 N 1+I 2o-Ni}JdF`$*ڐ DjQYvA{I.'6:>L)@va<+d;7$qGT67)Q9=sʾi.}i >_|Vpgg^*hiQ+ߴ mD>PS>"C]Cnʮuw 6ݾIvE9v!,y'Є R~x^"/Pj޴~ #۫/EL|jTnTRi )^Cvr(Nk Ev8]㼆Fw%EԜ +Zv 'ԕ" of,f8z[#iz4A2`Pل颂p hm= 8Z]2}w+hEWhقl7UD0 n*R=tF+P瞎«Ndd TJ$Xeƚ=ڸV$C+0I駗_=iKwt<|߹׆-z MgZj.TFSv@$F F8]Za}Nt--E 5DJSMAI[7F맰B>ǡrY DIl<.t"C*Ȁ *g/@j|3q^&y3IuB >ϭ/vv7Z@"j]8)@1TIMtb,P>1@줏atIK!kUR3 Jyw,m`o QQA ?.!8e[s6C tZ wBV*H6{*WM;Cs[ك+3ٻ }D|V1EWy) z7VT#Ay.\[{}y_9({q{$̶ +kOL 36֗F~ Bυϱ"˹+5pQژJWa1\pB*MqT,(ʸe"i`M:hy^>Q.UdNUvض( S,Tr(oך}W}-\+HSTDX(4u|/>~`?4/6O7XjL՞[[]K^? 40 agQX] *c c~.7IxKs03w`e'D wy[>~0U%aύ7pqBu5ˤc f!KAYmdI u)1T|#It)G@W+3P[ܫa2^t^ɀG7Hh{$ch@PUҳ9I:Ht9j*"!3@B$Ang}b9R\YЙ7Ư&!"|?*.>)y^2W.:[+>Đ4`nioxLRBCMM!m x۪iI}J%ZyPI0![Xq%23wé򹓡͵Tmf%x^3φYYV{3rp+,?r<yP@3j*Q{) Zmfna8}0vЮI٧5Ǖk{vm[vRW)KϤ ]<:m@ӞM2k?@}>en|"("C,4<5 ,G /V=9aݞ g%"JZ7C%=kG1LYhWvKrsbX֛BP?_^']EFIZ;%'b_,?$em>]FQN#0}~N]P5'أv".Rz4}&G[Mx>^x|8ZJxeBoI G yuF[nk$w0}Vrns \=N=D\|5P0Kj n6+\﹋i5nuWxE-B THC"+v R=3yl_l{?`F"xV|sCZ_M#hv=3 +jI^[;SN蒖ȝ>#Kf`hDrI1LVl,J&T>!<"PN$O)ģUvR#%pOo'>g'3P}ŀYzU~l+OT1J)IyP B/u?E71&!}qh6!bZzY2b',LxÒ,k7#jc!k֫>-A?ɓFz@?$c}8ʕn%hrUR`ƆV+j'3\,adb e;BTTp)qzj ]e5Mlə<|;8nh-V4hFSLy5+4 zzź#sSF-*E`jqP/D/.%rv,#s ;_~CAµy硬+#օѨKsVu6t8'pXPDϳ}f}~Y4O܋2)|Djt=c PmiM`y}FQs(XvYqgM1#y|ȄIUmNNMc^h7f՝ᒗ켲滁`3@۠S 6Ү=FǏpg2Wu~mS ɘlb?V [2oV;NCY]mby^&W7~8z#f[t!ӷ >U~N]ћ:gFtaBt,*]Z:dh?jǡBGV&ڪ.^(ūQEmɅRcKyU;"jPQ5u9>ö\tVKX0b/I^YY/ߏDߏ"G]^rޅZ_dJ Ŀ U~OMOkp`bxviEODQ-ܮ"x5lWfUy}$]M/.鱅ÙT{[B +2mEng2)tфw8?hjNq 0!␥pܪ"O:>Ϯ#Vwt Ȓ?U-^ Vd֘!! Č]I U֕ʞI27h"' * "11+Ķ}{Ei> E;%F0cBbr=& =%Ov޴Ի8n .M7WC_8<<9k5'] 5Ѥ UP'w8&[@ؖ=hq,q(r4WDt$'VlyL8WLEj! ~pppW'k WUJ2-E_mݴa U_ZKp\yJw>D'j~Y:mT1+րQg>Ր;h#(e1pP޳OI8y<LJ bPwL);`(VEY&Y &}#p*Rt)|[S?=|#<9Pӏ3/BN3rdBuן4q#ԱaVWDnYӻVU WZ9%QcQAfq>w ʶNDi)SV8<7 4+0ןI!"4b-t*(ݔX+IPu-=ҳ{jtfN.CpjymaN}c(BT)ߔy߿2Є )*[0EN [ܲƹpU̹}8sKʚ d?U/3ɅsqvP׎lb٤gj'>Bc>Py4t'JC(Sޯ[e㸱CȎcJX(ϜܐM,񰹺UۛM Ȗ8EX[gD;:.%pD;%JtjT\<:wV.*ڏ߸I05i޽gCa@uzDY~^Gx{tK铕0,LnrZ/xҁ."LfJչE0hK}]Rt&"gAz&1=4̍^$_>bA^ge@ j8*ec0=$d,ը^ä8D(|ѯ_GVUMr;7'^E^L4sw3%QV;@dXpX *bO&~oЋ?MrOndSVlu,a1$WtixV^į cW U羱1DHyy__##΀B},B%OoW-͞X%:Ѧc̞P+n}gfx8^.RmK1p1zW+)I ~0*Z\"Of6Z07Noeʣ#–Ec>ӭ:'41[!ƅ£,{MErѸmQG+ďV#_iO^[}Jԡ NGBr@~BX{/-pӜ~#oO*$\c+4OimiPi1#nsUmXćUgﺕ=TөXXBcZ3s.r2%ԡN,Sc&F,Kr:*rÙ_:r<󤘪wŕ"WE8_Z`+fT" L}aĠfMBg9^wU 9<+J?_\'<ͩǣoVTHM!l7yK05t-DCRX*H xKfEWΈLDHahyxvk-r"ԗfF+ [y- yv}zpNXԧX^gg -1mc {_'5CU,9ިDr26y{œK1x6|?z4D1{ ^*uOD9唱#Q_``%}wq۰`vvm%JJA@0>T@znrI(;;ǁ/ >(ÀFUV8~L4d>#m)j+ HA^[d=RuioW%Őjyj %flcA햍ݧԳ[ σSƑLx"3T&xSi/ƧГ>8@ozg*} E|Z>$ 9kZUWۓ142wv`SC {4NF5OmWL" |`[c9Ơu/ʊ@qy#N՟Ի2@JGM:,{]aȡAhFVfmZK.f,m7Ddz= x5VdR>S6[jx=@·n3>zB=8GCQhƒq]51)*gHHl0U~I3`;%W$:<(z aso:S 4Q"-}}&N;uBcyqOSZ.1h%c<ºCBѾsad| -/1v &_g[& {8fRs@*u}k<69ܘo>li6AĮ=H0#:FCjVc0gpWrb<!Z%&^+mo=1JMYE%sˡm!I q;"ғN]WѸ5"ї'X*: r^6/ͳε*< =h/AxY戞8:V SkДn91G/L^\X)*0͍_IN{U/bE{&q8;YӜ5ߟ0L IfLzOBbj[gacD//bZנAL[r7Ăl&H۰7-#$>̌956YC"u{XH֫HP@P~jf޽`06CjyM*,o1Lz" pk1Exm t9 %Iň[6prkI StM9!)rsvǿ:gWV-F 2c}# shXPNn=zv#{ `mLL&4cD)>!jNI יNx|̪QWՏ:{Yl w[5{BP;X\0G&Op-}"留t 8ֲpQ?!Ոn:qhi }72?Z@iHq VYYiJ3WFUB[#0YaEl%7[ 3ЇX5iE9w[8^m-O))سNJI7A)Pda{NFcZC_[W:۳hl$Pܦl釚ga "[X[p1+ےwEIΣw%u"#Qd~破o&dq$Oc. 50 jS5fP3%>.l~apR/aq=·&M$9>zfхX@Вڟ !wtf)J3Pc ?sذzŊT(- zP c'kX5`)^q)IDZM/ٗk*ر u̇zXӹoqPKأvs CҬ55@l f~@i$Mz}EPQS;"SؙD^ֳk*A%3>WeȾ, >wv$c+8=tWؘ-ɰ:;*fi+ur-/IS _C-z,GeqZuRgCpj0n4%9 @#WIMFhV2 @1 +YVir;,_(CC?S( qo&1l*}WAeCn v#(uSU#-N]+69oY3Hh5ڣ9wܑdr *<))B%{Hrd*"^q+H!O,yĔ9 q/|@ 2%XX mwDyWa:Ϛrik܄P3y{t '4?b.]N?$Psu1x8ثBJ)n"ܱn/iU|N4~cSSYqn`y+NZg lX}Oʝ wC޵_ٵ)= ,UƜuʸwʵ͊cX1,+F ~K)=شb/G t?lZE7!|5cS&K#`]A> 4gUe𛳽 [zJ'ї>d=s3_s6o NS.Ou/6z#b<[H;5l߼yt0׳R\ʯkh’D~7 i,Қ1ǧz3W+eIVMBHQNpO3mQJ`l > endobj 62 0 obj <> endobj 446 0 obj <> endobj 447 0 obj <> endobj xref 0 448 0000000000 65535 f 0000004834 00000 n 0000000015 00000 n 0000004649 00000 n 0000005030 00000 n 0000005106 00000 n 0000005233 00000 n 0000005309 00000 n 0000005436 00000 n 0000005512 00000 n 0000005639 00000 n 0000005716 00000 n 0000005844 00000 n 0000005921 00000 n 0000006049 00000 n 0000006126 00000 n 0000006254 00000 n 0000006331 00000 n 0000006459 00000 n 0000006536 00000 n 0000006664 00000 n 0000006741 00000 n 0000006869 00000 n 0000006946 00000 n 0000007075 00000 n 0000007152 00000 n 0000007280 00000 n 0000007357 00000 n 0000007485 00000 n 0000013228 00000 n 0000013385 00000 n 0000012687 00000 n 0000012532 00000 n 0000013100 00000 n 0000012970 00000 n 0000013542 00000 n 0000012842 00000 n 0000007622 00000 n 0000007886 00000 n 0000013703 00000 n 0000551553 00000 n 0000459480 00000 n 0000462915 00000 n 0000558371 00000 n 0000462560 00000 n 0000470225 00000 n 0000462536 00000 n 0000513090 00000 n 0000462512 00000 n 0000521673 00000 n 0000461872 00000 n 0000574390 00000 n 0000461396 00000 n 0000595182 00000 n 0000460711 00000 n 0000609319 00000 n 0000460241 00000 n 0000013898 00000 n 0000026189 00000 n 0000079988 00000 n 0000143773 00000 n 0000261231 00000 n 0000609656 00000 n 0000013805 00000 n 0000019897 00000 n 0000014116 00000 n 0000544503 00000 n 0000459911 00000 n 0000020052 00000 n 0000032322 00000 n 0000032392 00000 n 0000032357 00000 n 0000032548 00000 n 0000038582 00000 n 0000034403 00000 n 0000038325 00000 n 0000040380 00000 n 0000036286 00000 n 0000038187 00000 n 0000036148 00000 n 0000032139 00000 n 0000022949 00000 n 0000085099 00000 n 0000084943 00000 n 0000085135 00000 n 0000085177 00000 n 0000084760 00000 n 0000026040 00000 n 0000031883 00000 n 0000032010 00000 n 0000026465 00000 n 0000497283 00000 n 0000459115 00000 n 0000479918 00000 n 0000458751 00000 n 0000040518 00000 n 0000041932 00000 n 0000032590 00000 n 0000480149 00000 n 0000032795 00000 n 0000043346 00000 n 0000034540 00000 n 0000043377 00000 n 0000043588 00000 n 0000036580 00000 n 0000044016 00000 n 0000044230 00000 n 0000044608 00000 n 0000044829 00000 n 0000038773 00000 n 0000045115 00000 n 0000056518 00000 n 0000067789 00000 n 0000072547 00000 n 0000095733 00000 n 0000095576 00000 n 0000095770 00000 n 0000095813 00000 n 0000095392 00000 n 0000075947 00000 n 0000106213 00000 n 0000106407 00000 n 0000106250 00000 n 0000106444 00000 n 0000108096 00000 n 0000113871 00000 n 0000112027 00000 n 0000111888 00000 n 0000109982 00000 n 0000109843 00000 n 0000106029 00000 n 0000079836 00000 n 0000080267 00000 n 0000085411 00000 n 0000086826 00000 n 0000087037 00000 n 0000087397 00000 n 0000096048 00000 n 0000097463 00000 n 0000097674 00000 n 0000098034 00000 n 0000114010 00000 n 0000115425 00000 n 0000106487 00000 n 0000108235 00000 n 0000116840 00000 n 0000117051 00000 n 0000110281 00000 n 0000117467 00000 n 0000117683 00000 n 0000112264 00000 n 0000118028 00000 n 0000130851 00000 n 0000137787 00000 n 0000148852 00000 n 0000148889 00000 n 0000149046 00000 n 0000149549 00000 n 0000149339 00000 n 0000149089 00000 n 0000148668 00000 n 0000139227 00000 n 0000176841 00000 n 0000176915 00000 n 0000176878 00000 n 0000177072 00000 n 0000182925 00000 n 0000178724 00000 n 0000182651 00000 n 0000184704 00000 n 0000180610 00000 n 0000182512 00000 n 0000180471 00000 n 0000176657 00000 n 0000141517 00000 n 0000217455 00000 n 0000217492 00000 n 0000217418 00000 n 0000217649 00000 n 0000223344 00000 n 0000223048 00000 n 0000223187 00000 n 0000221047 00000 n 0000221186 00000 n 0000219299 00000 n 0000225373 00000 n 0000225234 00000 n 0000217234 00000 n 0000143631 00000 n 0000144040 00000 n 0000149764 00000 n 0000151179 00000 n 0000151395 00000 n 0000151770 00000 n 0000151988 00000 n 0000152294 00000 n 0000152504 00000 n 0000152832 00000 n 0000160515 00000 n 0000169387 00000 n 0000184843 00000 n 0000186258 00000 n 0000177115 00000 n 0000178863 00000 n 0000187673 00000 n 0000187884 00000 n 0000180905 00000 n 0000188319 00000 n 0000188533 00000 n 0000188849 00000 n 0000189066 00000 n 0000183097 00000 n 0000189325 00000 n 0000201169 00000 n 0000213807 00000 n 0000225545 00000 n 0000226960 00000 n 0000217692 00000 n 0000219438 00000 n 0000228375 00000 n 0000228589 00000 n 0000221440 00000 n 0000228919 00000 n 0000229118 00000 n 0000229346 00000 n 0000229557 00000 n 0000223627 00000 n 0000229984 00000 n 0000230201 00000 n 0000230460 00000 n 0000241671 00000 n 0000242610 00000 n 0000253485 00000 n 0000256912 00000 n 0000274737 00000 n 0000274700 00000 n 0000275505 00000 n 0000275721 00000 n 0000275152 00000 n 0000274774 00000 n 0000276192 00000 n 0000278856 00000 n 0000279088 00000 n 0000279320 00000 n 0000276997 00000 n 0000277816 00000 n 0000280323 00000 n 0000280721 00000 n 0000276796 00000 n 0000276580 00000 n 0000278648 00000 n 0000279522 00000 n 0000281106 00000 n 0000278441 00000 n 0000278234 00000 n 0000280130 00000 n 0000276381 00000 n 0000280525 00000 n 0000280907 00000 n 0000279941 00000 n 0000277617 00000 n 0000277418 00000 n 0000278032 00000 n 0000275990 00000 n 0000279709 00000 n 0000277186 00000 n 0000274579 00000 n 0000261092 00000 n 0000268324 00000 n 0000261497 00000 n 0000529925 00000 n 0000458160 00000 n 0000268456 00000 n 0000268585 00000 n 0000268843 00000 n 0000274470 00000 n 0000315437 00000 n 0000334456 00000 n 0000366633 00000 n 0000407011 00000 n 0000438632 00000 n 0000281458 00000 n 0000281495 00000 n 0000281404 00000 n 0000281352 00000 n 0000281322 00000 n 0000281538 00000 n 0000282750 00000 n 0000283924 00000 n 0000284133 00000 n 0000284350 00000 n 0000284564 00000 n 0000284775 00000 n 0000284812 00000 n 0000285059 00000 n 0000285326 00000 n 0000285566 00000 n 0000285820 00000 n 0000286154 00000 n 0000286420 00000 n 0000286730 00000 n 0000286981 00000 n 0000287232 00000 n 0000287482 00000 n 0000287729 00000 n 0000287986 00000 n 0000288243 00000 n 0000288528 00000 n 0000288821 00000 n 0000289132 00000 n 0000289375 00000 n 0000289657 00000 n 0000289960 00000 n 0000290245 00000 n 0000290530 00000 n 0000290773 00000 n 0000291069 00000 n 0000291344 00000 n 0000291589 00000 n 0000291847 00000 n 0000300355 00000 n 0000305639 00000 n 0000309524 00000 n 0000312625 00000 n 0000315332 00000 n 0000315671 00000 n 0000321308 00000 n 0000340728 00000 n 0000340771 00000 n 0000340489 00000 n 0000325536 00000 n 0000344324 00000 n 0000344367 00000 n 0000344085 00000 n 0000329217 00000 n 0000372058 00000 n 0000372101 00000 n 0000371817 00000 n 0000331401 00000 n 0000375721 00000 n 0000375764 00000 n 0000375480 00000 n 0000334280 00000 n 0000334761 00000 n 0000341169 00000 n 0000341508 00000 n 0000344751 00000 n 0000345092 00000 n 0000347481 00000 n 0000379318 00000 n 0000379361 00000 n 0000379078 00000 n 0000352850 00000 n 0000382936 00000 n 0000382979 00000 n 0000382695 00000 n 0000357277 00000 n 0000385999 00000 n 0000386042 00000 n 0000385755 00000 n 0000361245 00000 n 0000389395 00000 n 0000389438 00000 n 0000389152 00000 n 0000366426 00000 n 0000371685 00000 n 0000366967 00000 n 0000505601 00000 n 0000457861 00000 n 0000372491 00000 n 0000372840 00000 n 0000376152 00000 n 0000376507 00000 n 0000379763 00000 n 0000380098 00000 n 0000383355 00000 n 0000383691 00000 n 0000386428 00000 n 0000386776 00000 n 0000389826 00000 n 0000390180 00000 n 0000392768 00000 n 0000412742 00000 n 0000412785 00000 n 0000412495 00000 n 0000396230 00000 n 0000415883 00000 n 0000415926 00000 n 0000415639 00000 n 0000399813 00000 n 0000418920 00000 n 0000418963 00000 n 0000418680 00000 n 0000403273 00000 n 0000422237 00000 n 0000422280 00000 n 0000421997 00000 n 0000406831 00000 n 0000407321 00000 n 0000413171 00000 n 0000413490 00000 n 0000416310 00000 n 0000416630 00000 n 0000419357 00000 n 0000419687 00000 n 0000422666 00000 n 0000423014 00000 n 0000425580 00000 n 0000444308 00000 n 0000444351 00000 n 0000444062 00000 n 0000428677 00000 n 0000447665 00000 n 0000447708 00000 n 0000447419 00000 n 0000431536 00000 n 0000450928 00000 n 0000450971 00000 n 0000450681 00000 n 0000434649 00000 n 0000454518 00000 n 0000454561 00000 n 0000454271 00000 n 0000438474 00000 n 0000438920 00000 n 0000444749 00000 n 0000445082 00000 n 0000448104 00000 n 0000448432 00000 n 0000451357 00000 n 0000451705 00000 n 0000454947 00000 n 0000455295 00000 n 0000463161 00000 n 0000470430 00000 n 0000488273 00000 n 0000497571 00000 n 0000505819 00000 n 0000513300 00000 n 0000521968 00000 n 0000530138 00000 n 0000544847 00000 n 0000551798 00000 n 0000558665 00000 n 0000574775 00000 n 0000595728 00000 n 0000609730 00000 n 0000609778 00000 n trailer <]/Root 446 0 R/Size 448>> startxref 610065 %%EOF genomicsdb-0.0~git20231212.9d7ddd0/doc/mainpage.dox000066400000000000000000000034141453617025200214110ustar00rootroot00000000000000/*! * @mainpage TileDB * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section About * * TileDB is a new system for efficient management of scientific data, i.e., of * massive quantities of data that are naturally represented by * multi-dimensional arrays. TileDB is written in C++ for Linux Ubuntu, CentOS * and Mac OS X, and open-sourced under the MIT license. The current release * includes only the TileDB storage manager module exposed as a C API library, * which makes it easy for programmers to write applications for diverse, * complex, parallel, scientific data analytics. * */ genomicsdb-0.0~git20231212.9d7ddd0/examples/000077500000000000000000000000001453617025200201635ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/examples/CMakeLists.txt000066400000000000000000000051471453617025200227320ustar00rootroot00000000000000# # examples/CMakeLists.txt # # # The MIT License # # Copyright (c) 2016 MIT and Intel Corporation # Copyright (c) 2018-2019 Omics Data Automation, Inc. # Copyright (c) 2023 dātma, inc™ # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Function that builds an executable per example function(build_TileDB_example TARGET) add_executable(${TARGET} EXCLUDE_FROM_ALL src/${TARGET}.cc) target_link_libraries(${TARGET} tiledb_static ${TILEDB_LIB_DEPENDENCIES}) endfunction() # Get the example sources file(GLOB TILEDB_EXAMPLE_SOURCES "src/*.cc") # Include TileDB C API headers include_directories("${CMAKE_SOURCE_DIR}/core/include/c_api/") # Initialize name for example binaries set(EXAMPLE_BINS) # Iterate over all example sources and call the build function foreach(EXAMPLE_SOURCE ${TILEDB_EXAMPLE_SOURCES}) # Get the binary name get_filename_component(EXAMPLE_STRIPPED ${EXAMPLE_SOURCE} NAME) STRING(REGEX REPLACE ".cc$" "" EXAMPLE_BIN ${EXAMPLE_STRIPPED}) set(EXAMPLE_BINS ${EXAMPLE_BINS} ${EXAMPLE_BIN}) # Build example executable build_TileDB_example(${EXAMPLE_BIN}) endforeach() # Add custom target 'examples' add_custom_target(examples DEPENDS tiledb_static) add_dependencies(examples ${EXAMPLE_BINS}) # Run a subset of examples for Continuous Integration Testing if(NOT TILEDB_DISABLE_TESTING) set(CI_TESTS ${CMAKE_SOURCE_DIR}/examples/run_examples.sh && echo "Running diff with expected_results" && diff log ${CMAKE_SOURCE_DIR}/examples/expected_results) add_test(NAME ci_tests COMMAND ${CI_TESTS}) add_custom_target(ci_tests COMMAND ${CI_TESTS} DEPENDS examples) add_dependencies(tests ci_tests) endif() genomicsdb-0.0~git20231212.9d7ddd0/examples/expected_results000066400000000000000000000241631453617025200234760ustar00rootroot00000000000000Example 1: Running ./tiledb_workspace_group_create... Example 1: Done running ./tiledb_workspace_group_create Example 2: Running ./tiledb_ls_workspaces... my_workspace Example 2: Done running ./tiledb_ls_workspaces Example 3: Running ./tiledb_array_create_dense... Example 3: Done running ./tiledb_array_create_dense Example 4: Running ./tiledb_array_create_sparse... Example 4: Done running ./tiledb_array_create_sparse Example 5: Running ./tiledb_array_primitive... Array name: my_workspace/dense_arrays/my_array_A Attributes: a1 a2 a3 The array is dense Array name: my_workspace/sparse_arrays/my_array_B Attributes: a1 a2 a3 The array is sparse Example 5: Done running ./tiledb_array_primitive Example 6: Running ./tiledb_array_write_dense_1... Example 6: Done running ./tiledb_array_write_dense_1 Example 7: Running ./tiledb_array_write_sparse_1... Example 7: Done running ./tiledb_array_write_sparse_1 Example 8: Running ./tiledb_ls... my_workspace WORKSPACE Example 8: Done running ./tiledb_ls Example 9: Running ./tiledb_array_read_dense_1... a1 a2 (a3.first, a3.second) ----------------------------------------- 0 a ( 0.1, 0.2) 1 bb ( 1.1, 1.2) 2 ccc ( 2.1, 2.2) 3 dddd ( 3.1, 3.2) 4 e ( 4.1, 4.2) 5 ff ( 5.1, 5.2) 6 ggg ( 6.1, 6.2) 7 hhhh ( 7.1, 7.2) 8 i ( 8.1, 8.2) 9 jj ( 9.1, 9.2) 10 kkk ( 10.1, 10.2) 11 llll ( 11.1, 11.2) 12 m ( 12.1, 12.2) 13 nn ( 13.1, 13.2) 14 ooo ( 14.1, 14.2) 15 pppp ( 15.1, 15.2) Example 9: Done running ./tiledb_array_read_dense_1 Example 10: Running ./tiledb_array_write_dense_2... Example 10: Done running ./tiledb_array_write_dense_2 Example 11: Running ./tiledb_array_read_dense_1... a1 a2 (a3.first, a3.second) ----------------------------------------- 0 a ( 0.1, 0.2) 1 bb ( 1.1, 1.2) 2 ccc ( 2.1, 2.2) 3 dddd ( 3.1, 3.2) 4 e ( 4.1, 4.2) 5 ff ( 5.1, 5.2) 6 ggg ( 6.1, 6.2) 7 hhhh ( 7.1, 7.2) 8 i ( 8.1, 8.2) 9 jj ( 9.1, 9.2) 10 kkk ( 10.1, 10.2) 11 llll ( 11.1, 11.2) 12 m ( 12.1, 12.2) 13 nn ( 13.1, 13.2) 14 ooo ( 14.1, 14.2) 15 pppp ( 15.1, 15.2) Example 11: Done running ./tiledb_array_read_dense_1 Example 12: Running ./tiledb_array_read_dense_2... a1 ---- Reading cells... 9 11 12 Reading cells... 13 14 15 Example 12: Done running ./tiledb_array_read_dense_2 Example 13: Running ./tiledb_array_write_dense_sorted... Example 13: Done running ./tiledb_array_write_dense_sorted Example 14: Running ./tiledb_array_read_dense_1... a1 a2 (a3.first, a3.second) ----------------------------------------- 0 a ( 0.1, 0.2) 1 bb ( 1.1, 1.2) 2 ccc ( 2.1, 2.2) 3 dddd ( 3.1, 3.2) 4 e ( 4.1, 4.2) 5 ff ( 5.1, 5.2) 6 ggg ( 6.1, 6.2) 7 hhhh ( 7.1, 7.2) 8 i ( 8.1, 8.2) 9 jj ( 9.1, 9.2) 10 kkk ( 10.1, 10.2) 11 llll ( 11.1, 11.2) 12 m ( 12.1, 12.2) 13 nn ( 13.1, 13.2) 14 ooo ( 14.1, 14.2) 15 pppp ( 15.1, 15.2) Example 14: Done running ./tiledb_array_read_dense_1 Example 15: Running ./tiledb_array_update_dense_1... Example 15: Done running ./tiledb_array_update_dense_1 Example 16: Running ./tiledb_array_read_dense_1... a1 a2 (a3.first, a3.second) ----------------------------------------- 0 a ( 0.1, 0.2) 1 bb ( 1.1, 1.2) 2 ccc ( 2.1, 2.2) 3 dddd ( 3.1, 3.2) 4 e ( 4.1, 4.2) 5 ff ( 5.1, 5.2) 6 ggg ( 6.1, 6.2) 7 hhhh ( 7.1, 7.2) 8 i ( 8.1, 8.2) 9 jj ( 9.1, 9.2) 10 kkk ( 10.1, 10.2) 11 llll ( 11.1, 11.2) 112 M (112.1, 112.2) 113 NN (113.1, 113.2) 114 OOO (114.1, 114.2) 115 PPPP (115.1, 115.2) Example 16: Done running ./tiledb_array_read_dense_1 Example 17: Running ./tiledb_array_update_dense_2... Example 17: Done running ./tiledb_array_update_dense_2 Example 18: Running ./tiledb_array_read_dense_1... a1 a2 (a3.first, a3.second) ----------------------------------------- 0 a ( 0.1, 0.2) 1 bb ( 1.1, 1.2) 2 ccc ( 2.1, 2.2) 3 dddd ( 3.1, 3.2) 4 e ( 4.1, 4.2) 5 ff ( 5.1, 5.2) 6 ggg ( 6.1, 6.2) 7 hhhh ( 7.1, 7.2) 208 u (208.1, 208.2) 9 jj ( 9.1, 9.2) 10 kkk ( 10.1, 10.2) 211 wwww (211.1, 211.2) 212 x (212.1, 212.2) 213 yy (213.1, 213.2) 114 OOO (114.1, 114.2) 115 PPPP (115.1, 115.2) Example 18: Done running ./tiledb_array_read_dense_1 Example 19: Running ./tiledb_array_write_sparse_1... Example 19: Done running ./tiledb_array_write_sparse_1 Example 20: Running ./tiledb_array_read_sparse_1... coords a1 a2 (a3.first, a3.second) -------------------------------------------------- (1, 1) 0 a ( 0.1, 0.2) (1, 2) 1 bb ( 1.1, 1.2) (1, 4) 2 ccc ( 2.1, 2.2) (2, 3) 3 dddd ( 3.1, 3.2) (3, 1) 4 e ( 4.1, 4.2) (4, 2) 5 ff ( 5.1, 5.2) (3, 3) 6 ggg ( 6.1, 6.2) (3, 4) 7 hhhh ( 7.1, 7.2) Example 20: Done running ./tiledb_array_read_sparse_1 Example 21: Running ./tiledb_array_read_sparse_filter_1... coords a1 a2 (a3.first, a3.second) -------------------------------------------------- Example 21: Done running ./tiledb_array_read_sparse_filter_1 Example 22: Running ./tiledb_array_iterator_sparse... a1 ---- 5 6 7 Example 22: Done running ./tiledb_array_iterator_sparse Example 23: Running ./tiledb_array_iterator_sparse_filter... a1 ---- 6 7 Example 23: Done running ./tiledb_array_iterator_sparse_filter Example 24: Running ./tiledb_array_read_sparse_2... a1 ---- Reading cells... 5 6 Reading cells... 7 Example 24: Done running ./tiledb_array_read_sparse_2 Example 25: Running ./tiledb_array_read_sparse_filter_2... a1 ---- Reading cells... 6 Reading cells... 7 Example 25: Done running ./tiledb_array_read_sparse_filter_2 Example 26: Running ./tiledb_array_write_sparse_2... Example 26: Done running ./tiledb_array_write_sparse_2 Example 27: Running ./tiledb_array_read_sparse_1... coords a1 a2 (a3.first, a3.second) -------------------------------------------------- (1, 1) 0 a ( 0.1, 0.2) (1, 2) 1 bb ( 1.1, 1.2) (1, 4) 2 ccc ( 2.1, 2.2) (2, 3) 3 dddd ( 3.1, 3.2) (3, 1) 4 e ( 4.1, 4.2) (4, 2) 5 ff ( 5.1, 5.2) (3, 3) 6 ggg ( 6.1, 6.2) (3, 4) 7 hhhh ( 7.1, 7.2) Example 27: Done running ./tiledb_array_read_sparse_1 Example 28: Running ./tiledb_array_read_sparse_2... a1 ---- Reading cells... 5 6 Reading cells... 7 Example 28: Done running ./tiledb_array_read_sparse_2 Example 29: Running ./tiledb_array_read_sparse_filter_2... a1 ---- Reading cells... 6 Reading cells... 7 Example 29: Done running ./tiledb_array_read_sparse_filter_2 Example 30: Running ./tiledb_array_update_sparse_1... Example 30: Done running ./tiledb_array_update_sparse_1 Example 31: Running ./tiledb_array_read_sparse_1... coords a1 a2 (a3.first, a3.second) -------------------------------------------------- (1, 1) 0 a ( 0.1, 0.2) (1, 2) 1 bb ( 1.1, 1.2) (1, 4) 2 ccc ( 2.1, 2.2) (2, 3) 3 dddd ( 3.1, 3.2) (3, 1) 4 e ( 4.1, 4.2) (3, 2) 104 u (104.1, 104.2) (4, 1) 105 vvvv (105.1, 105.2) (4, 2) 5 ff ( 5.1, 5.2) (3, 3) 106 w (106.1, 106.2) (3, 4) 107 yyy (107.1, 107.2) Example 31: Done running ./tiledb_array_read_sparse_1 Example 32: Running ./tiledb_array_read_sparse_filter_1... coords a1 a2 (a3.first, a3.second) -------------------------------------------------- (4, 1) 105 vvvv (105.1, 105.2) Example 32: Done running ./tiledb_array_read_sparse_filter_1 Example 33: Running ./tiledb_array_read_sparse_filter_2... a1 ---- Reading cells... Reading cells... 106 107 Example 33: Done running ./tiledb_array_read_sparse_filter_2 Example 34: Running ./tiledb_array_iterator_sparse... a1 ---- 104 5 106 107 Example 34: Done running ./tiledb_array_iterator_sparse Example 35: Running ./tiledb_array_iterator_sparse_filter... a1 ---- 104 106 107 Example 35: Done running ./tiledb_array_iterator_sparse_filter Example 36: Running ./tiledb_array_read_sparse_2... a1 ---- Reading cells... 104 5 Reading cells... 106 107 Example 36: Done running ./tiledb_array_read_sparse_2 Example 37: Running ./tiledb_array_consolidate... Example 37: Done running ./tiledb_array_consolidate Example 38: Running ./tiledb_array_read_dense_1... a1 a2 (a3.first, a3.second) ----------------------------------------- 0 a ( 0.1, 0.2) 1 bb ( 1.1, 1.2) 2 ccc ( 2.1, 2.2) 3 dddd ( 3.1, 3.2) 4 e ( 4.1, 4.2) 5 ff ( 5.1, 5.2) 6 ggg ( 6.1, 6.2) 7 hhhh ( 7.1, 7.2) 208 u (208.1, 208.2) 9 jj ( 9.1, 9.2) 10 kkk ( 10.1, 10.2) 211 wwww (211.1, 211.2) 212 x (212.1, 212.2) 213 yy (213.1, 213.2) 114 OOO (114.1, 114.2) 115 PPPP (115.1, 115.2) Example 38: Done running ./tiledb_array_read_dense_1 Example 39: Running ./tiledb_array_read_sparse_1... coords a1 a2 (a3.first, a3.second) -------------------------------------------------- (1, 1) 0 a ( 0.1, 0.2) (1, 2) 1 bb ( 1.1, 1.2) (1, 4) 2 ccc ( 2.1, 2.2) (2, 3) 3 dddd ( 3.1, 3.2) (3, 1) 4 e ( 4.1, 4.2) (3, 2) 104 u (104.1, 104.2) (4, 1) 105 vvvv (105.1, 105.2) (4, 2) 5 ff ( 5.1, 5.2) (3, 3) 106 w (106.1, 106.2) (3, 4) 107 yyy (107.1, 107.2) Example 39: Done running ./tiledb_array_read_sparse_1 Example 40: Running ./tiledb_array_read_dense_2... a1 ---- Reading cells... 9 211 212 Reading cells... 213 114 115 Example 40: Done running ./tiledb_array_read_dense_2 Example 41: Running ./tiledb_array_read_sparse_2... a1 ---- Reading cells... 104 5 Reading cells... 106 107 Example 41: Done running ./tiledb_array_read_sparse_2 Example 42: Running ./tiledb_array_3d... 4 results coords a1 a2 ----------------------- 0, 0, 0 0 first 0, 0, 1 1 second 0, 2, 3 2 third 2, 1, 1 3 fourth Example 42: Done running ./tiledb_array_3d genomicsdb-0.0~git20231212.9d7ddd0/examples/run_examples.sh000077500000000000000000000107221453617025200232260ustar00rootroot00000000000000#!/bin/bash # # run_examples.sh # # # The MIT License # # Copyright (c) 2018 Omics Data Automation Inc. and Intel Corporation # Copyright (c) 2019 Omics Data Automation Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # This runs a small subset of example binaries from /examples/ directory. # Usage without arguments(workspace will be a default "my_workspace" in # Results are either in a "log" file for the default case or .log # if path is specified. # e.g for ./run_examples.sh gs://my_bucket/my_dir/my_test, expect results in test.log # Check the log file against the /examples/expected_results file. check_rc() { if [[ $# -eq 1 ]]; then if [[ $1 -ne 0 ]]; then echo echo "Exit Status=$1. Quitting execution of run_examples.sh" exit $1 fi fi } run_example() { if [[ $# -eq 3 ]] then logfile=`basename $2`.log echo "Example $3: Running $1..." | tee -a ${logfile} $1 $2 | tee -a ${logfile} check_rc ${PIPESTATUS[0]} echo "Example $3: Done running $1" | tee -a ${logfile} else echo "Example $2: Running $1..." | tee -a log $1 | tee -a log check_rc ${PIPESTATUS[0]} echo "Example $2: Done running $1" | tee -a log fi } if [[ -n $1 ]] then rm -fr `basename $1`.log else rm -fr log fi run_example ./tiledb_workspace_group_create $1 1 run_example ./tiledb_ls_workspaces $1 2 run_example ./tiledb_array_create_dense $1 3 run_example ./tiledb_array_create_sparse $1 4 run_example ./tiledb_array_primitive $1 5 run_example ./tiledb_array_write_dense_1 $1 6 run_example ./tiledb_array_write_sparse_1 $1 7 sleep 5 run_example ./tiledb_ls $1 8 run_example ./tiledb_array_read_dense_1 $1 9 run_example ./tiledb_array_write_dense_2 $1 10 sleep 5 run_example ./tiledb_array_read_dense_1 $1 11 run_example ./tiledb_array_read_dense_2 $1 12 run_example ./tiledb_array_write_dense_sorted $1 13 sleep 5 run_example ./tiledb_array_read_dense_1 $1 14 run_example ./tiledb_array_update_dense_1 $1 15 sleep 5 run_example ./tiledb_array_read_dense_1 $1 16 run_example ./tiledb_array_update_dense_2 $1 17 sleep 5 run_example ./tiledb_array_read_dense_1 $1 18 run_example ./tiledb_array_write_sparse_1 $1 19 sleep 5 run_example ./tiledb_array_read_sparse_1 $1 20 run_example ./tiledb_array_read_sparse_filter_1 $1 21 run_example ./tiledb_array_iterator_sparse $1 22 run_example ./tiledb_array_iterator_sparse_filter $1 23 run_example ./tiledb_array_read_sparse_2 $1 24 run_example ./tiledb_array_read_sparse_filter_2 $1 25 run_example ./tiledb_array_write_sparse_2 $1 26 sleep 5 run_example ./tiledb_array_read_sparse_1 $1 27 run_example ./tiledb_array_read_sparse_2 $1 28 run_example ./tiledb_array_read_sparse_filter_2 $1 29 run_example ./tiledb_array_update_sparse_1 $1 30 sleep 5 run_example ./tiledb_array_read_sparse_1 $1 31 run_example ./tiledb_array_read_sparse_filter_1 $1 32 run_example ./tiledb_array_read_sparse_filter_2 $1 33 run_example ./tiledb_array_iterator_sparse $1 34 run_example ./tiledb_array_iterator_sparse_filter $1 35 run_example ./tiledb_array_read_sparse_2 $1 36 run_example ./tiledb_array_consolidate $1 37 sleep 5 run_example ./tiledb_array_read_dense_1 $1 38 run_example ./tiledb_array_read_sparse_1 $1 39 run_example ./tiledb_array_read_dense_2 $1 40 run_example ./tiledb_array_read_sparse_2 $1 41 sleep 5 run_example ./tiledb_array_3d $1 42 genomicsdb-0.0~git20231212.9d7ddd0/examples/src/000077500000000000000000000000001453617025200207525ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/examples/src/examples.h000066400000000000000000000037511453617025200227470ustar00rootroot00000000000000/** * @file examples.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018 Omics Data Automation Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Examples Header File handling errors of TileDB api */ // TODO: move all examples to do error checking #ifndef TILEDB_EXAMPLES_H #define TILEDB_EXAMPLES_H #include "tiledb.h" #include "tiledb_storage.h" #include #include #define CHECK_RC(...) \ do { \ int rc = __VA_ARGS__; \ if (rc) { \ printf("%s", &tiledb_errmsg[0]); \ printf("[Examples::%s] Runtime Error.\n", __FILE__); \ return rc; \ } \ } while (false) #endif /* TILEDB_EXAMPLES_H */ genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_3d.cc000066400000000000000000000203331453617025200243110ustar00rootroot00000000000000/** * @file tiledb_array_3d.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to create, write, and read a sparse 3d array. */ #include "examples.h" int main(int argc, char *argv[]) { // ================================== ARRAY CREATE ====================== // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Prepare parameters for array schema const char* array_name = "my_workspace/sparse_arrays/my_3d_array"; // Delete the array if it exists if (is_dir(tiledb_ctx, array_name)) { CHECK_RC(tiledb_delete(tiledb_ctx, array_name)); } const char* attributes[] = { "a1", "a2" }; // Two attributes const char* dimensions[] = { "d1", "d2", "d3" }; // Three dimensions int64_t domain[] = { 0, 4, // d1 0, 4, // d2 0, 4 // d3 }; const int cell_val_num[] = { 1, // a1 TILEDB_VAR_NUM // a2 }; const int compression[] = { TILEDB_GZIP +TILEDB_BIT_SHUFFLE, // a1 TILEDB_GZIP, // a2 TILEDB_GZIP +TILEDB_DELTA_ENCODE // coordinates }; const int offsets_compression[] = { 0, // a1 - DON'T CARE TILEDB_GZIP +TILEDB_DELTA_ENCODE, // a2 }; int64_t tile_extents[] = { 1, // d1 1, // d2 1 // d3 }; const int types[] = { TILEDB_INT32, // a1 TILEDB_CHAR, // a2 TILEDB_INT64 // coordinates }; // Set array schema TileDB_ArraySchema array_schema; tiledb_array_set_schema( &array_schema, // Array schema struct array_name, // Array name attributes, // Attributes 2, // Number of attributes 5, // Capacity TILEDB_ROW_MAJOR, // Cell order cell_val_num, // Number of cell values per attribute compression, // Compression NULL, // Compression level - Use defaults offsets_compression, // Offsets compression NULL, // Offsets compression level 0, // Sparse array dimensions, // Dimensions 3, // Number of dimensions domain, // Domain 6*sizeof(int64_t), // Domain length in bytes tile_extents, // Tile extents 3*sizeof(int64_t), // Tile extents length in bytes TILEDB_ROW_MAJOR, // Tile order types // Types ); // Create array CHECK_RC(tiledb_array_create(tiledb_ctx, &array_schema)); // Free array schema CHECK_RC(tiledb_array_free_schema(&array_schema)); /* Finalize context. */ CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); // ================================== ARRAY WRITE ====================== if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_WRITE, // Mode NULL, // Entire domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 0, 1, 2, 3 }; size_t buffer_a2[] = { 0, 5, 11, 16 }; const char buffer_var_a2[] = "firstsecondthirdfourth"; int64_t buffer_coords[] = { 0, 0, 0, 0, 0, 1, 0, 2, 3, 2, 1, 1 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_coords) }; // Write to array CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Finalize the array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Free array schema CHECK_RC(tiledb_array_free_schema(&array_schema)); /* Finalize context. */ CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); // ================================== ARRAY READ ====================== if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_READ, // Mode NULL, // Whole domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int r_buffer_a1[10]; size_t r_buffer_a2[10]; char r_buffer_var_a2[30]; int64_t r_buffer_coords[30]; void* r_buffers[] = { r_buffer_a1, r_buffer_a2, r_buffer_var_a2, r_buffer_coords }; size_t r_buffer_sizes[] = { sizeof(r_buffer_a1), sizeof(r_buffer_a2), sizeof(r_buffer_var_a2), sizeof(r_buffer_coords) }; // Read from array CHECK_RC(tiledb_array_read(tiledb_array, r_buffers, r_buffer_sizes)); // Print cell values int64_t result_num = r_buffer_sizes[0] / sizeof(int); printf("%ld results\n", (long)result_num); printf("coords\t a1\t a2\n"); printf("-----------------------\n"); for(int i=0; i #include // Simply prints the input string to stdout void *print_upon_completion(void* s) { printf("%s\n", (char*) s); return NULL; } int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_READ, // Mode NULL, // Whole domain NULL, // All attributes 0); // Number of attributes // Prepare subarray int64_t subarray[] = { 3, 4, 2, 4 }; // [3,4] on first dim, [2,4] on second // Prepare cell buffers int buffer_a1[16]; size_t buffer_a2[16]; char buffer_var_a2[40]; float buffer_a3[32]; void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2), sizeof(buffer_a3) }; // Prepare AIO request TileDB_AIO_Request tiledb_aio_request; memset(&tiledb_aio_request, 0, sizeof(struct TileDB_AIO_Request)); tiledb_aio_request.buffers_ = buffers; tiledb_aio_request.buffer_sizes_ = buffer_sizes; tiledb_aio_request.subarray_ = subarray; tiledb_aio_request.completion_handle_ = print_upon_completion; char s[100] = "AIO request completed"; tiledb_aio_request.completion_data_ = s; // Read from array tiledb_array_aio_read(tiledb_array, &tiledb_aio_request); // Wait for AIO to complete printf("AIO in progress\n"); while(tiledb_aio_request.status_ != TILEDB_AIO_COMPLETED); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); printf(" a1\t a2\t (a3.first, a3.second)\n"); printf("-----------------------------------------\n"); for(int i=0; i // Simply prints the input string to stdout void *print_upon_completion(void* s) { printf("%s\n", (char*) s); return NULL; } int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_WRITE, // Mode NULL, // Entire domain NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 0, 1, 2, 3, // Upper left tile 4, 5, 6, 7, // Upper right tile 8, 9, 10, 11, // Lower left tile 12, 13, 14, 15 // Lower right tile }; size_t buffer_a2[] = { 0, 1, 3, 6, // Upper left tile 10, 11, 13, 16, // Upper right tile 20, 21, 23, 26, // Lower left tile 30, 31, 33, 36 // Lower right tile }; char buffer_var_a2[] = "abbcccdddd" // Upper left tile "effggghhhh" // Upper right tile "ijjkkkllll" // Lower left tile "mnnooopppp"; // Lower right tile float buffer_a3[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2, // Upper left tile 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2, // Upper right tile 8.1, 8.2, 9.1, 9.2, 10.1, 10.2, 11.1, 11.2, // Lower left tile 12.1, 12.2, 13.1, 13.2, 14.1, 14.2, 15.1, 15.2, // Lower right tile }; void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3) }; // Prepare AIO request TileDB_AIO_Request tiledb_aio_request; // ALWAYS zero out the struct before populating it memset(&tiledb_aio_request, 0, sizeof(struct TileDB_AIO_Request)); tiledb_aio_request.buffers_ = buffers; tiledb_aio_request.buffer_sizes_ = buffer_sizes; tiledb_aio_request.completion_handle_ = print_upon_completion; char s[100] = "AIO request completed"; tiledb_aio_request.completion_data_ = s; // Write to array tiledb_array_aio_write(tiledb_array, &tiledb_aio_request); // Wait for AIO to complete printf("AIO in progress\n"); while(tiledb_aio_request.status_ != TILEDB_AIO_COMPLETED); // Finalize array tiledb_array_finalize(tiledb_array); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_consolidate.cc000066400000000000000000000040141453617025200263050ustar00rootroot00000000000000/** * @file tiledb_array_consolidate.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to consolidate arrays. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Consolidate the dense array CHECK_RC(tiledb_array_consolidate(tiledb_ctx, "my_workspace/dense_arrays/my_array_A")); // Consolidate the sparse array CHECK_RC(tiledb_array_consolidate(tiledb_ctx, "my_workspace/sparse_arrays/my_array_B")); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_create_dense.cc000066400000000000000000000111001453617025200264140ustar00rootroot00000000000000/** * @file tiledb_array_create_dense.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to create a dense array. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Prepare parameters for array schema const char* array_name = "my_workspace/dense_arrays/my_array_A"; const char* attributes[] = { "a1", "a2", "a3" }; // Three attributes const char* dimensions[] = { "d1", "d2" }; // Two dimensions int64_t domain[] = { 1, 4, // d1 1, 4 // d2 }; const int cell_val_num[] = { 1, // a1 TILEDB_VAR_NUM, // a2 2 // a3 }; const int compression[] = { TILEDB_GZIP, // a1 #ifdef ENABLE_ZSTD TILEDB_ZSTD, // a2 #else TILEDB_GZIP, // a2 #endif #ifdef ENABLE_LZ4 TILEDB_LZ4, // a3 #else TILEDB_GZIP, // a3 #endif TILEDB_NO_COMPRESSION // coordinates }; const int offsets_compression[] = { 0, // a1 - DON'T CARE TILEDB_GZIP, // a2 0 // a3 - DON'T CARE }; int64_t tile_extents[] = { 2, // d1 2 // d2 }; const int types[] = { TILEDB_INT32, // a1 TILEDB_CHAR, // a2 TILEDB_FLOAT32, // a3 TILEDB_INT64 // coordinates }; // Set array schema TileDB_ArraySchema array_schema; CHECK_RC(tiledb_array_set_schema( &array_schema, // Array schema struct array_name, // Array name attributes, // Attributes 3, // Number of attributes 2, // Capacity TILEDB_ROW_MAJOR, // Cell order cell_val_num, // Number of cell values per attribute compression, // Compression NULL, // Compression level, use defaults offsets_compression, // Offsets compression NULL, // Offsets compression level, use defaults 1, // Dense array dimensions, // Dimensions 2, // Number of dimensions domain, // Domain 4*sizeof(int64_t), // Domain length in bytes tile_extents, // Tile extents 2*sizeof(int64_t), // Tile extents length in bytes TILEDB_ROW_MAJOR, // Tile order types // Types )); // Create array CHECK_RC(tiledb_array_create(tiledb_ctx, &array_schema)); // Free array schema CHECK_RC(tiledb_array_free_schema(&array_schema)); /* Finalize context. */ CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_create_sparse.cc000066400000000000000000000110201453617025200266140ustar00rootroot00000000000000/** * @file tiledb_array_create_sparse.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to create a sparse array. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Prepare parameters for array schema const char* array_name = "my_workspace/sparse_arrays/my_array_B"; const char* attributes[] = { "a1", "a2", "a3" }; // Three attributes const char* dimensions[] = { "d1", "d2" }; // Two dimensions int64_t domain[] = { 1, 4, // d1 1, 4 // d2 }; const int cell_val_num[] = { 1, // a1 TILEDB_VAR_NUM, // a2 2 // a3 }; const int compression[] = { TILEDB_GZIP +TILEDB_BIT_SHUFFLE, // a1 #ifdef ENABLE_BLOSC TILEDB_BLOSC, // a2 #else TILEDB_GZIP, // a2 #endif TILEDB_NO_COMPRESSION, // a3 TILEDB_GZIP +TILEDB_DELTA_ENCODE // coordinates }; const int offsets_compression[] = { 0, // a1 - DON'T CARE TILEDB_GZIP +TILEDB_DELTA_ENCODE, // a2 0 // a3 - DON'T CARE }; int64_t tile_extents[] = { 2, // d1 2 // d2 }; const int types[] = { TILEDB_INT32, // a1 TILEDB_CHAR, // a2 TILEDB_FLOAT32, // a3 TILEDB_INT64 // coordinates }; // Set array schema TileDB_ArraySchema array_schema; tiledb_array_set_schema( &array_schema, // Array schema struct array_name, // Array name attributes, // Attributes 3, // Number of attributes 2, // Capacity TILEDB_ROW_MAJOR, // Cell order cell_val_num, // Number of cell values per attribute compression, // Compression NULL, // Compression level - Use defaults offsets_compression, // Offsets compression NULL, // Offsets compression level 0, // Sparse array dimensions, // Dimensions 2, // Number of dimensions domain, // Domain 4*sizeof(int64_t), // Domain length in bytes tile_extents, // Tile extents 2*sizeof(int64_t), // Tile extents length in bytes TILEDB_ROW_MAJOR, // Tile order types // Types ); // Create array CHECK_RC(tiledb_array_create(tiledb_ctx, &array_schema)); // Free array schema CHECK_RC(tiledb_array_free_schema(&array_schema)); /* Finalize context. */ CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_iterator_dense.cc000066400000000000000000000065051453617025200270170ustar00rootroot00000000000000/** * @file tiledb_array_iterator_dense.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to use an iterator for dense arrays. */ #include "tiledb.h" #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Prepare cell buffers int buffer_a1[3]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_ArrayIterator* tiledb_array_it; tiledb_array_iterator_init( tiledb_ctx, // Context &tiledb_array_it, // Array iterator "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1, // Number of attributes buffers, // Buffers used internally buffer_sizes); // Buffer sizes // Iterate over all values in subarray printf(" a1\n----\n"); const int* a1_v; size_t a1_size; while(!tiledb_array_iterator_end(tiledb_array_it)) { // Get value tiledb_array_iterator_get_value( tiledb_array_it, // Array iterator 0, // Attribute id (const void**) &a1_v,// Value &a1_size); // Value size (useful in variable-sized attributes) printf("%3d\n", *a1_v); // Advance iterator tiledb_array_iterator_next(tiledb_array_it); } // Finalize array tiledb_array_iterator_finalize(tiledb_array_it); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_iterator_sparse.cc000066400000000000000000000066251453617025200272210ustar00rootroot00000000000000/** * @file tiledb_array_iterator_sparse.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to use an iterator for sparse arrays. */ #include "tiledb.h" #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Prepare cell buffers int buffer_a1[3]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_ArrayIterator* tiledb_array_it; tiledb_array_iterator_init( tiledb_ctx, // Context &tiledb_array_it, // Array iterator "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1, // Number of attributes buffers, // Buffers used internally buffer_sizes); // Buffer sizes // Iterate over all values in subarray printf(" a1\n----\n"); const int* a1_v; size_t a1_size; while(!tiledb_array_iterator_end(tiledb_array_it)) { // Get value tiledb_array_iterator_get_value( tiledb_array_it, // Array iterator 0, // Attribute id (const void**) &a1_v,// Value &a1_size); // Value size (useful in variable-sized attributes) // Print value (if not a deletion) if(*a1_v != TILEDB_EMPTY_INT32) printf("%3d\n", *a1_v); // Advance iterator tiledb_array_iterator_next(tiledb_array_it); } // Finalize array tiledb_array_iterator_finalize(tiledb_array_it); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_iterator_sparse_filter.cc000066400000000000000000000067201453617025200305620ustar00rootroot00000000000000/** * @file tiledb_array_iterator_sparse_filter.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2019, 2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to use an iterator with a filter for sparse arrays. */ #include "tiledb.h" #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Prepare cell buffers int buffer_a1[3]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_ArrayIterator* tiledb_array_it; tiledb_array_iterator_init_with_filter( tiledb_ctx, // Context &tiledb_array_it, // Array iterator "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1, // Number of attributes buffers, // Buffers used internally buffer_sizes, // Buffer sizes "a1 > 5"); // Iterate over all values in subarray printf(" a1\n----\n"); const int* a1_v; size_t a1_size; while(!tiledb_array_iterator_end(tiledb_array_it)) { // Get value tiledb_array_iterator_get_value( tiledb_array_it, // Array iterator 0, // Attribute id (const void**) &a1_v,// Value &a1_size); // Value size (useful in variable-sized attributes) // Print value (if not a deletion) if(*a1_v != TILEDB_EMPTY_INT32) printf("%3d\n", *a1_v); // Advance iterator tiledb_array_iterator_next(tiledb_array_it); } // Finalize array tiledb_array_iterator_finalize(tiledb_array_it); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_consolidate_dense.cc000066400000000000000000000141271453617025200313450ustar00rootroot00000000000000/** * @file tiledb_array_parallel_consolidate_dense.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to mix reading and consolidation on a dense array in parallel. * This program works with pthreads, but it is very easy to write a similar * program for OpenMP. */ #include "tiledb.h" #include #include // The consolidation function to be computed in parallel void *parallel_consolidate(void* args); // The read function to be computed in parallel void *parallel_read(void* args); // The arguments for each invocation of parallel_write typedef struct _thread_data_t { const TileDB_CTX* tiledb_ctx; const char* array_name; const void* subarray; void** buffers; size_t* buffer_sizes; int count; } thread_data_t; int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/dense_arrays/my_array_A"; // Prepare cell buffers // --- Upper left tile --- const int64_t subarray_1[] = { 1, 2, 1, 2 }; int buffer_a1_1[4]; void* buffers_1[] = { buffer_a1_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 1, 2, 3, 4 }; int buffer_a1_2[4]; void* buffers_2[] = { buffer_a1_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2) }; // --- Lower left tile --- const int64_t subarray_3[] = { 3, 4, 1, 2 }; int buffer_a1_3[4]; void* buffers_3[] = { buffer_a1_3 }; size_t buffer_sizes_3[] = { sizeof(buffer_a1_3) }; // --- Lower right tile --- const int64_t subarray_4[] = { 3, 4, 3, 4 }; int buffer_a1_4[4]; void* buffers_4[] = { buffer_a1_4 }; size_t buffer_sizes_4[] = { sizeof(buffer_a1_4) }; // Initialize 4 pthreads for reading, plus 1 for consolidation pthread_t threads[5]; thread_data_t thread_data[5]; // Write in parallel for(int i=0; i<5; ++i) { // Populate the thread data thread_data[i].tiledb_ctx = tiledb_ctx; thread_data[i].array_name = array_name; if(i==0) { // First tile thread_data[i].buffers = buffers_1; thread_data[i].buffer_sizes = buffer_sizes_1; thread_data[i].subarray = subarray_1; } else if(i==1) { // Second tile thread_data[i].buffers = buffers_2; thread_data[i].buffer_sizes = buffer_sizes_2; thread_data[i].subarray = subarray_2; } else if(i==2) { // Third tile thread_data[i].buffers = buffers_3; thread_data[i].buffer_sizes = buffer_sizes_3; thread_data[i].subarray = subarray_3; } else if(i==3) { // Fourth tile thread_data[i].buffers = buffers_4; thread_data[i].buffer_sizes = buffer_sizes_4; thread_data[i].subarray = subarray_4; } // Create thread if(i>=0 && i<4) pthread_create(&threads[i], NULL, parallel_read, &thread_data[i]); else // i == 4 pthread_create(&threads[i], NULL, parallel_consolidate, &thread_data[i]); } // Wait till all threads finish for(int i=0; i<5; ++i) pthread_join(threads[i], NULL); // Output result int total_count = 0; for(int i=0; i<4; ++i) total_count += thread_data[i].count; printf("Number of a1 values greater than 10: %d \n", total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void *parallel_consolidate(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Consolidate array printf("Started consolidation\n"); tiledb_array_consolidate(data->tiledb_ctx, data->array_name); printf("Finished consolidation\n"); return 0; } void *parallel_read(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( data->tiledb_ctx, // Context &tiledb_array, // Array object data->array_name, // Array name TILEDB_ARRAY_READ, // Mode data->subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array printf("Started reading\n"); tiledb_array_read(tiledb_array, data->buffers, data->buffer_sizes); printf("Finished reading\n"); // Count number of a1 values greater than 10 data->count = 0; int* a1 = (int*) data->buffers[0]; int num = data->buffer_sizes[0] / sizeof(int); for(int i=0; i 10) ++data->count; } // Finalize array tiledb_array_finalize(tiledb_array); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_consolidate_sparse.cc000066400000000000000000000125531453617025200315450ustar00rootroot00000000000000/** * @file tiledb_array_parallel_consolidate_sparse.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to mix reading and consolidation on a sparse array in parallel. * This program works with pthreads, but it is very easy to write a similar * program for OpenMP. */ #include "tiledb.h" #include #include // The consolidation function to be computed in parallel void *parallel_consolidate(void* args); // The function to be computed in parallel void *parallel_read(void* args); // The arguments for each invocation of parallel_write typedef struct _thread_data_t { const TileDB_CTX* tiledb_ctx; const char* array_name; const void* subarray; void** buffers; size_t* buffer_sizes; int count; } thread_data_t; int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/sparse_arrays/my_array_B"; // Prepare cell buffers // --- First read --- const int64_t subarray_1[] = { 1, 2, 1, 4 }; int buffer_a1_1[10]; void* buffers_1[] = { buffer_a1_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 3, 4, 1, 4 }; int buffer_a1_2[10]; void* buffers_2[] = { buffer_a1_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2) }; // Initialize 2 pthreads for reading, plus 1 for consolidation pthread_t threads[3]; thread_data_t thread_data[3]; // Write in parallel for(int i=0; i<3; ++i) { // Populate the thread data thread_data[i].tiledb_ctx = tiledb_ctx; thread_data[i].array_name = array_name; if(i==0) { // First read thread_data[i].buffers = buffers_1; thread_data[i].buffer_sizes = buffer_sizes_1; thread_data[i].subarray = subarray_1; } else if(i==1) { // Second read thread_data[i].buffers = buffers_2; thread_data[i].buffer_sizes = buffer_sizes_2; thread_data[i].subarray = subarray_2; } // Create thread if(i>=0 && i<2) pthread_create(&threads[i], NULL, parallel_read, &thread_data[i]); else // i == 2 pthread_create(&threads[i], NULL, parallel_consolidate, &thread_data[i]); } // Wait till all threads finish for(int i=0; i<3; ++i) pthread_join(threads[i], NULL); // Output result int total_count = 0; for(int i=0; i<2; ++i) total_count += thread_data[i].count; printf("Number of a1 values greater than 5: %d \n", total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void *parallel_consolidate(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Consolidate array printf("Started consolidation\n"); tiledb_array_consolidate(data->tiledb_ctx, data->array_name); printf("Finished consolidation\n"); return 0; } void *parallel_read(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( data->tiledb_ctx, // Context &tiledb_array, // Array object data->array_name, // Array name TILEDB_ARRAY_READ, // Mode data->subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array printf("Started reading\n"); tiledb_array_read(tiledb_array, data->buffers, data->buffer_sizes); printf("Finished reading\n"); // Count number of a1 values greater than 10 data->count = 0; int* a1 = (int*) data->buffers[0]; int num = data->buffer_sizes[0] / sizeof(int); for(int i=0; i 5) ++data->count; // Finalize array tiledb_array_finalize(tiledb_array); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_read_dense_1.cc000066400000000000000000000126311453617025200301720ustar00rootroot00000000000000/** * @file tiledb_array_parallel_read_dense_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from a dense array in parallel with pthreads. */ #include "tiledb.h" #include #include // The function to be computed in parallel void *parallel_read(void* args); // The arguments for each invocation of parallel_write typedef struct _thread_data_t { const TileDB_CTX* tiledb_ctx; const char* array_name; const void* subarray; void** buffers; size_t* buffer_sizes; int count; } thread_data_t; int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/dense_arrays/my_array_A"; // Prepare cell buffers // --- Upper left tile --- const int64_t subarray_1[] = { 1, 2, 1, 2 }; int buffer_a1_1[4]; void* buffers_1[] = { buffer_a1_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 1, 2, 3, 4 }; int buffer_a1_2[4]; void* buffers_2[] = { buffer_a1_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2) }; // --- Lower left tile --- const int64_t subarray_3[] = { 3, 4, 1, 2 }; int buffer_a1_3[4]; void* buffers_3[] = { buffer_a1_3 }; size_t buffer_sizes_3[] = { sizeof(buffer_a1_3) }; // --- Lower right tile --- const int64_t subarray_4[] = { 3, 4, 3, 4 }; int buffer_a1_4[4]; void* buffers_4[] = { buffer_a1_4 }; size_t buffer_sizes_4[] = { sizeof(buffer_a1_4) }; // Initialize 4 pthreads and corresponding data pthread_t threads[4]; thread_data_t thread_data[4]; // Read in parallel for(int i=0; i<4; ++i) { // Populate the thread data thread_data[i].tiledb_ctx = tiledb_ctx; thread_data[i].array_name = array_name; if(i==0) { // First tile thread_data[i].buffers = buffers_1; thread_data[i].buffer_sizes = buffer_sizes_1; thread_data[i].subarray = subarray_1; } else if(i==1) { // Second tile thread_data[i].buffers = buffers_2; thread_data[i].buffer_sizes = buffer_sizes_2; thread_data[i].subarray = subarray_2; } else if(i==2) { // Third tile thread_data[i].buffers = buffers_3; thread_data[i].buffer_sizes = buffer_sizes_3; thread_data[i].subarray = subarray_3; } else if(i==3) { // Fourth tile thread_data[i].buffers = buffers_4; thread_data[i].buffer_sizes = buffer_sizes_4; thread_data[i].subarray = subarray_4; } // Create thread pthread_create(&threads[i], NULL, parallel_read, &thread_data[i]); } // Wait till all threads finish for(int i=0; i<4; ++i) pthread_join(threads[i], NULL); // Output result int total_count = 0; for(int i=0; i<4; ++i) total_count += thread_data[i].count; printf("Number of a1 values greater than 10: %d \n", total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void *parallel_read(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( data->tiledb_ctx, // Context &tiledb_array, // Array object data->array_name, // Array name TILEDB_ARRAY_READ, // Mode data->subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array tiledb_array_read(tiledb_array, data->buffers, data->buffer_sizes); // Count number of a1 values greater than 10 data->count = 0; int* a1 = (int*) data->buffers[0]; int num = data->buffer_sizes[0] / sizeof(int); for(int i=0; i 10) ++data->count; } // Finalize array tiledb_array_finalize(tiledb_array); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_read_dense_2.cc000066400000000000000000000122401453617025200301670ustar00rootroot00000000000000/** * @file tiledb_array_parallel_read_dense_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from a dense array in parallel with OpenMP. */ #include "tiledb.h" #include #ifdef HAVE_OPENMP #include // The function to be computed in parallel void parallel_read( const TileDB_CTX* tiledb_ctx, const char* array_name, const void* subarray, void** buffers, size_t* buffer_sizes, int* count); int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/dense_arrays/my_array_A"; // Prepare cell buffers // --- Upper left tile --- const int64_t subarray_1[] = { 1, 2, 1, 2 }; int buffer_a1_1[4]; void* buffers_1[] = { buffer_a1_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 1, 2, 3, 4 }; int buffer_a1_2[4]; void* buffers_2[] = { buffer_a1_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2) }; // --- Lower left tile --- const int64_t subarray_3[] = { 3, 4, 1, 2 }; int buffer_a1_3[4]; void* buffers_3[] = { buffer_a1_3 }; size_t buffer_sizes_3[] = { sizeof(buffer_a1_3) }; // --- Lower right tile --- const int64_t subarray_4[] = { 3, 4, 3, 4 }; int buffer_a1_4[4]; void* buffers_4[] = { buffer_a1_4 }; size_t buffer_sizes_4[] = { sizeof(buffer_a1_4) }; // Buffer to store the individual thread counts int counts[4]; // Write in parallel #pragma omp parallel for for(int i=0; i<4; ++i) { // Populate the thread data void** buffers; size_t* buffer_sizes; const void* subarray; if(i==0) { // First tile buffers = buffers_1; buffer_sizes = buffer_sizes_1; subarray = subarray_1; } else if(i==1) { // Second tile buffers = buffers_2; buffer_sizes = buffer_sizes_2; subarray = subarray_2; } else if(i==2) { // Third tile buffers = buffers_3; buffer_sizes = buffer_sizes_3; subarray = subarray_3; } else if(i==3) { // Fourth tile buffers = buffers_4; buffer_sizes = buffer_sizes_4; subarray = subarray_4; } // Parallel read parallel_read( tiledb_ctx, array_name, subarray, buffers, buffer_sizes, &counts[i]); } // Output result int total_count = 0; for(int i=0; i<4; ++i) total_count += counts[i]; printf("Number of a1 values greater than 10: %d \n", total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void parallel_read( const TileDB_CTX* tiledb_ctx, const char* array_name, const void* subarray, void** buffers, size_t* buffer_sizes, int* count) { // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_READ, // Mode subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Count number of a1 values greater than 10 *count = 0; int* a1 = (int*) buffers[0]; int num = buffer_sizes[0] / sizeof(int); for(int i=0; i 10) ++(*count); // Finalize array tiledb_array_finalize(tiledb_array); } #else int main() { printf("OpenMP not supported.\n"); return 0; } #endif genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_read_mpi_io_dense.cc000066400000000000000000000077421453617025200313150ustar00rootroot00000000000000/** * @file tiledb_array_read_mpi_io_dense.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from a dense array in parallel with MPI, activating * also the MPI-IO read mode (although the latter is optional - the user * could alternatively use mmap or standard OS read). Note that the case * of sparse arrays is similar. */ #include "tiledb.h" #include #include #ifdef HAVE_MPI #include int main(int argc, char** argv) { // Initialize MPI and get rank MPI_Init(&argc, &argv); int rank; MPI_Comm mpi_comm = MPI_COMM_WORLD; MPI_Comm_rank(mpi_comm, &rank); // Properly set the configuration parameters TileDB_Config tiledb_config; memset(&tiledb_config, 0, sizeof(struct TileDB_Config)); tiledb_config.read_method_ = TILEDB_IO_MPI; // Activate MPI-IO tiledb_config.mpi_comm_ = &mpi_comm; // Initialize context with the default configuration parameters TileDB_CTX* tiledb_ctx; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); // Array name const char* array_name = "my_workspace/dense_arrays/my_array_A"; // Prepare cell buffers // --- Upper left tile --- const int64_t subarray_0[] = { 1, 2, 1, 2 }; // --- Upper right tile --- const int64_t subarray_1[] = { 1, 2, 3, 4 }; // --- Lower left tile --- const int64_t subarray_2[] = { 3, 4, 1, 2 }; // --- Lower right tile --- const int64_t subarray_3[] = { 3, 4, 3, 4 }; // Set buffers int buffer[4]; void* buffers[] = { buffer }; size_t buffer_sizes[] = { sizeof(buffer) }; // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Choose subarray based on rank const int64_t* subarray; if(rank == 0) subarray = subarray_0; else if(rank == 1) subarray = subarray_1; else if(rank == 2) subarray = subarray_2; else if(rank == 3) subarray = subarray_3; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_READ, // Mode subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); // Output result int total_count = 0; for(int i=0; i<4; ++i) if(buffer[i] > 10) ++total_count; printf("Process %d: Number of a1 values greater " "than 10: %d \n", rank, total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); // Finalize MPI MPI_Finalize(); return 0; } #else int main() { printf("MPI not supported.\n"); return 0; } #endif genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_read_sparse_1.cc000066400000000000000000000112651453617025200303730ustar00rootroot00000000000000/** * @file tiledb_array_parallel_read_sparse_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from a sparse array in parallel with pthreads. */ #include "tiledb.h" #include #include // The function to be computed in parallel void *parallel_read(void* args); // The arguments for each invocation of parallel_write typedef struct _thread_data_t { const TileDB_CTX* tiledb_ctx; const char* array_name; const void* subarray; void** buffers; size_t* buffer_sizes; int count; } thread_data_t; int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/sparse_arrays/my_array_B"; // Prepare cell buffers // --- First read --- const int64_t subarray_1[] = { 1, 2, 1, 4 }; int buffer_a1_1[10]; void* buffers_1[] = { buffer_a1_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 3, 4, 1, 4 }; int buffer_a1_2[10]; void* buffers_2[] = { buffer_a1_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2) }; // Initialize 2 pthreads and corresponding data pthread_t threads[2]; thread_data_t thread_data[2]; // Write in parallel for(int i=0; i<2; ++i) { // Populate the thread data thread_data[i].tiledb_ctx = tiledb_ctx; thread_data[i].array_name = array_name; if(i==0) { // First read thread_data[i].buffers = buffers_1; thread_data[i].buffer_sizes = buffer_sizes_1; thread_data[i].subarray = subarray_1; } else if(i==1) { // Second read thread_data[i].buffers = buffers_2; thread_data[i].buffer_sizes = buffer_sizes_2; thread_data[i].subarray = subarray_2; } // Create thread pthread_create(&threads[i], NULL, parallel_read, &thread_data[i]); } // Wait till all threads finish for(int i=0; i<2; ++i) pthread_join(threads[i], NULL); // Output result int total_count = 0; for(int i=0; i<2; ++i) total_count += thread_data[i].count; printf("Number of a1 values greater than 5: %d \n", total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void *parallel_read(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( data->tiledb_ctx, // Context &tiledb_array, // Array object data->array_name, // Array name TILEDB_ARRAY_READ, // Mode data->subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array tiledb_array_read(tiledb_array, data->buffers, data->buffer_sizes); // Count number of a1 values greater than 10 data->count = 0; int* a1 = (int*) data->buffers[0]; int num = data->buffer_sizes[0] / sizeof(int); for(int i=0; i 5) ++data->count; // Finalize array tiledb_array_finalize(tiledb_array); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_read_sparse_2.cc000066400000000000000000000110211453617025200303620ustar00rootroot00000000000000/** * @file tiledb_array_parallel_read_sparse_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from a sparse array in parallel with OpenMP. */ #include "tiledb.h" #include #ifdef HAVE_OPENMP #include // The function to be computed in parallel void parallel_read( const TileDB_CTX* tiledb_ctx, const char* array_name, const void* subarray, void** buffers, size_t* buffer_sizes, int* count); int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/sparse_arrays/my_array_B"; // Prepare cell buffers // --- First read --- const int64_t subarray_1[] = { 1, 2, 1, 4 }; int buffer_a1_1[4]; void* buffers_1[] = { buffer_a1_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1) }; // --- Second read --- const int64_t subarray_2[] = { 3, 4, 1, 4 }; int buffer_a1_2[4]; void* buffers_2[] = { buffer_a1_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2) }; // Buffer to store the individual thread counts int counts[2]; // Write in parallel #pragma omp parallel for for(int i=0; i<2; ++i) { // Populate the thread data void** buffers; size_t* buffer_sizes; const void* subarray; if(i==0) { // First read buffers = buffers_1; buffer_sizes = buffer_sizes_1; subarray = subarray_1; } else if(i==1) { // Second read buffers = buffers_2; buffer_sizes = buffer_sizes_2; subarray = subarray_2; } // Parallel read parallel_read( tiledb_ctx, array_name, subarray, buffers, buffer_sizes, &counts[i]); } // Output result int total_count = 0; for(int i=0; i<2; ++i) total_count += counts[i]; printf("Number of a1 values greater than 5: %d \n", total_count); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void parallel_read( const TileDB_CTX* tiledb_ctx, const char* array_name, const void* subarray, void** buffers, size_t* buffer_sizes, int* count) { // Only attribute "a1" is needed const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_READ, // Mode subarray, // Subarray attributes, // Subset on attributes 1); // Number of attributes // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Count number of a1 values greater than 10 *count = 0; int* a1 = (int*) buffers[0]; int num = buffer_sizes[0] / sizeof(int); for(int i=0; i 5) ++(*count); // Finalize array tiledb_array_finalize(tiledb_array); } #else int main() { printf("OpenMP not supported."); return 0; } #endif genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_write_dense_1.cc000066400000000000000000000146401453617025200304130ustar00rootroot00000000000000/** * @file tiledb_array_parallel_write_dense_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a dense array in parallel with pthreads. */ #include "tiledb.h" #include // The function to be computed in parallel void *parallel_write(void* args); // The arguments for each invocation of parallel_write typedef struct _thread_data_t { const TileDB_CTX* tiledb_ctx; const char* array_name; const void* subarray; const void** buffers; const size_t* buffer_sizes; } thread_data_t; int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/dense_arrays/my_array_A"; // Prepare cell buffers // --- Upper left tile --- const int64_t subarray_1[] = { 1, 2, 1, 2 }; int buffer_a1_1[] = { 0, 1, 2, 3 }; size_t buffer_a2_1[] = { 0, 1, 3, 6 }; const char buffer_var_a2_1[] = "abbcccdddd"; float buffer_a3_1[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2 }; const void* buffers_1[] = { buffer_a1_1, buffer_a2_1, buffer_var_a2_1, buffer_a3_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1), sizeof(buffer_a2_1), sizeof(buffer_var_a2_1)-1, // No need to store the last '\0' character sizeof(buffer_a3_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 1, 2, 3, 4 }; int buffer_a1_2[] = { 4, 5, 6, 7 }; size_t buffer_a2_2[] = { 0, 1, 3, 6 }; const char buffer_var_a2_2[] = "effggghhhh"; float buffer_a3_2[] = { 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2 }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2), sizeof(buffer_a2_2), sizeof(buffer_var_a2_2)-1, // No need to store the last '\0' character sizeof(buffer_a3_2) }; // --- Lower left tile --- const int64_t subarray_3[] = { 3, 4, 1, 2 }; int buffer_a1_3[] = { 8, 9, 10, 11 }; size_t buffer_a2_3[] = { 0, 1, 3, 6 }; const char buffer_var_a2_3[] = "ijjkkkllll"; float buffer_a3_3[] = { 8.1, 8.2, 9.1, 9.2, 10.1, 10.2, 11.1, 11.2 }; const void* buffers_3[] = { buffer_a1_3, buffer_a2_3, buffer_var_a2_3, buffer_a3_3 }; size_t buffer_sizes_3[] = { sizeof(buffer_a1_3), sizeof(buffer_a2_3), sizeof(buffer_var_a2_3)-1, // No need to store the last '\0' character sizeof(buffer_a3_3) }; // --- Lower right tile --- const int64_t subarray_4[] = { 3, 4, 3, 4 }; int buffer_a1_4[] = { 12, 13, 14, 15 }; size_t buffer_a2_4[] = { 0, 1, 3, 6 }; const char buffer_var_a2_4[] = "mnnooopppp"; float buffer_a3_4[] = { 12.1, 12.2, 13.1, 13.2, 14.1, 14.2, 15.1, 15.2 }; const void* buffers_4[] = { buffer_a1_4, buffer_a2_4, buffer_var_a2_4, buffer_a3_4 }; size_t buffer_sizes_4[] = { sizeof(buffer_a1_4), sizeof(buffer_a2_4), sizeof(buffer_var_a2_4)-1, // No need to store the last '\0' character sizeof(buffer_a3_4) }; // Initialize 4 pthreads and corresponding data pthread_t threads[4]; thread_data_t thread_data[4]; // Write in parallel for(int i=0; i<4; ++i) { // Populate the thread data thread_data[i].tiledb_ctx = tiledb_ctx; thread_data[i].array_name = array_name; if(i==0) { // First tile thread_data[i].buffers = buffers_1; thread_data[i].buffer_sizes = buffer_sizes_1; thread_data[i].subarray = subarray_1; } else if(i==1) { // Second tile thread_data[i].buffers = buffers_2; thread_data[i].buffer_sizes = buffer_sizes_2; thread_data[i].subarray = subarray_2; } else if(i==2) { // Third tile thread_data[i].buffers = buffers_3; thread_data[i].buffer_sizes = buffer_sizes_3; thread_data[i].subarray = subarray_3; } else if(i==3) { // Fourth tile thread_data[i].buffers = buffers_4; thread_data[i].buffer_sizes = buffer_sizes_4; thread_data[i].subarray = subarray_4; } // Create thread pthread_create(&threads[i], NULL, parallel_write, &thread_data[i]); } // Wait till all threads finish for(int i=0; i<4; ++i) pthread_join(threads[i], NULL); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void *parallel_write(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( data->tiledb_ctx, // Context &tiledb_array, // Array object data->array_name, // Array name TILEDB_ARRAY_WRITE, // Mode data->subarray, // Subarray NULL, // All attributes 0); // Number of attributes // Write to array tiledb_array_write(tiledb_array, data->buffers, data->buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_write_dense_2.cc000066400000000000000000000141761453617025200304200ustar00rootroot00000000000000/** * @file tiledb_array_parallel_write_dense_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a dense array in parallel with OpenMP. */ #include "tiledb.h" #ifdef HAVE_OPENMP #include // The function to be computed in parallel void parallel_write( const TileDB_CTX* tiledb_ctx, const char* array_name, const void* subarray, const void** buffers, const size_t* buffer_sizes); int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Array name const char* array_name = "my_workspace/dense_arrays/my_array_A"; // Prepare cell buffers // --- Upper left tile --- const int64_t subarray_1[] = { 1, 2, 1, 2 }; int buffer_a1_1[] = { 0, 1, 2, 3 }; size_t buffer_a2_1[] = { 0, 1, 3, 6 }; const char buffer_var_a2_1[] = "abbcccdddd"; float buffer_a3_1[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2 }; const void* buffers_1[] = { buffer_a1_1, buffer_a2_1, buffer_var_a2_1, buffer_a3_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1), sizeof(buffer_a2_1), sizeof(buffer_var_a2_1)-1, // No need to store the last '\0' character sizeof(buffer_a3_1) }; // --- Upper right tile --- const int64_t subarray_2[] = { 1, 2, 3, 4 }; int buffer_a1_2[] = { 4, 5, 6, 7 }; size_t buffer_a2_2[] = { 0, 1, 3, 6 }; const char buffer_var_a2_2[] = "effggghhhh"; float buffer_a3_2[] = { 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2 }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2), sizeof(buffer_a2_2), sizeof(buffer_var_a2_2)-1, // No need to store the last '\0' character sizeof(buffer_a3_2) }; // --- Lower left tile --- const int64_t subarray_3[] = { 3, 4, 1, 2 }; int buffer_a1_3[] = { 8, 9, 10, 11 }; size_t buffer_a2_3[] = { 0, 1, 3, 6 }; const char buffer_var_a2_3[] = "ijjkkkllll"; float buffer_a3_3[] = { 8.1, 8.2, 9.1, 9.2, 10.1, 10.2, 11.1, 11.2 }; const void* buffers_3[] = { buffer_a1_3, buffer_a2_3, buffer_var_a2_3, buffer_a3_3 }; size_t buffer_sizes_3[] = { sizeof(buffer_a1_3), sizeof(buffer_a2_3), sizeof(buffer_var_a2_3)-1, // No need to store the last '\0' character sizeof(buffer_a3_3) }; // --- Lower right tile --- const int64_t subarray_4[] = { 3, 4, 3, 4 }; int buffer_a1_4[] = { 12, 13, 14, 15 }; size_t buffer_a2_4[] = { 0, 1, 3, 6 }; const char buffer_var_a2_4[] = "mnnooopppp"; float buffer_a3_4[] = { 12.1, 12.2, 13.1, 13.2, 14.1, 14.2, 15.1, 15.2 }; const void* buffers_4[] = { buffer_a1_4, buffer_a2_4, buffer_var_a2_4, buffer_a3_4 }; size_t buffer_sizes_4[] = { sizeof(buffer_a1_4), sizeof(buffer_a2_4), sizeof(buffer_var_a2_4)-1, // No need to store the last '\0' character sizeof(buffer_a3_4) }; // Write in parallel #pragma omp parallel for for(int i=0; i<4; ++i) { // Populate thread data const void** buffers; const size_t* buffer_sizes; const void* subarray; if(i==0) { // First tile buffers = buffers_1; buffer_sizes = buffer_sizes_1; subarray = subarray_1; } else if(i==1) { // Second tile buffers = buffers_2; buffer_sizes = buffer_sizes_2; subarray = subarray_2; } else if(i==2) { // Third tile buffers = buffers_3; buffer_sizes = buffer_sizes_3; subarray = subarray_3; } else if(i==3) { // Fourth tile buffers = buffers_4; buffer_sizes = buffer_sizes_4; subarray = subarray_4; } // Write parallel_write( tiledb_ctx, array_name, subarray, buffers, buffer_sizes); } // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void parallel_write( const TileDB_CTX* tiledb_ctx, const char* array_name, const void* subarray, const void** buffers, const size_t* buffer_sizes) { // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_WRITE, // Mode subarray, // Subarray NULL, // All attributes 0); // Number of attributes // Write to array tiledb_array_write(tiledb_array, buffers, buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); } #else #include int main() { printf("OpenMP not supported.\n"); return 0; } #endif genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_write_sparse_1.cc000066400000000000000000000114271453617025200306120ustar00rootroot00000000000000/** * @file tiledb_array_parallel_write_sparse_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a sparse array in parallel with pthreads. */ #include "tiledb.h" #include // The function to be computed in parallel void *parallel_write(void* args); // The arguments for each invocation of parallel_write typedef struct _thread_data_t { const TileDB_CTX* tiledb_ctx; const char* array_name; const void** buffers; const size_t* buffer_sizes; } thread_data_t; int main() { // Initialize context with the default configuration parameters TileDB_CTX* tiledb_ctx; tiledb_ctx_init(&tiledb_ctx, NULL); // Array name const char* array_name = "my_workspace/sparse_arrays/my_array_B"; // Prepare cell buffers // --- First write --- int buffer_a1_1[] = { 7, 5, 0 }; size_t buffer_a2_1[] = { 0, 4, 6 }; const char buffer_var_a2_1[] = "hhhhffa"; float buffer_a3_1[] = { 7.1, 7.2, 5.1, 5.2, 0.1, 0.2 }; int64_t buffer_coords_1[] = { 3, 4, 4, 2, 1, 1 }; const void* buffers_1[] = { buffer_a1_1, buffer_a2_1, buffer_var_a2_1, buffer_a3_1, buffer_coords_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1), sizeof(buffer_a2_1), sizeof(buffer_var_a2_1)-1, // No need to store the last '\0' character sizeof(buffer_a3_1), sizeof(buffer_coords_1) }; // --- Second write --- int buffer_a1_2[] = { 6, 4, 3, 1, 2 }; size_t buffer_a2_2[] = { 0, 3, 4, 8, 10 }; const char buffer_var_a2_2[] = "gggeddddbbccc"; float buffer_a3_2[] = { 6.1, 6.2, 4.1, 4.2, 3.1, 3.2, 1.1, 1.2, 2.1, 2.2 }; int64_t buffer_coords_2[] = { 3, 3, 3, 1, 2, 3, 1, 2, 1, 4 }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2, buffer_coords_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2), sizeof(buffer_a2_2), sizeof(buffer_var_a2_2)-1, // No need to store the last '\0' character sizeof(buffer_a3_2), sizeof(buffer_coords_2) }; // Initialize 2 pthreads and corresponding data pthread_t threads[2]; thread_data_t thread_data[2]; // Write in parallel for(int i=0; i<2; ++i) { // Populate the thread data thread_data[i].tiledb_ctx = tiledb_ctx; thread_data[i].array_name = array_name; if(i==0) { // First tile thread_data[i].buffers = buffers_1; thread_data[i].buffer_sizes = buffer_sizes_1; } else if(i==1) { // Second tile thread_data[i].buffers = buffers_2; thread_data[i].buffer_sizes = buffer_sizes_2; } // Create thread pthread_create(&threads[i], NULL, parallel_write, &thread_data[i]); } // Wait till all threads finish for(int i=0; i<2; ++i) pthread_join(threads[i], NULL); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void *parallel_write(void* args) { // Get arguments thread_data_t* data = (thread_data_t*) args; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( data->tiledb_ctx, // Context &tiledb_array, // Array object data->array_name, // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Inapplicable NULL, // All attributes 0); // Number of attributes // Write to array tiledb_array_write(tiledb_array, data->buffers, data->buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_parallel_write_sparse_2.cc000066400000000000000000000110521453617025200306050ustar00rootroot00000000000000/** * @file tiledb_array_parallel_write_sparse_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a sparse array in parallel with OpenMP. */ #include "tiledb.h" #ifdef HAVE_OPENMP #include // The function to be computed in parallel void parallel_write( const TileDB_CTX* tiledb_ctx, const char* array_name, const void** buffers, const size_t* buffer_sizes); int main() { // Initialize context with the default configuration parameters TileDB_CTX* tiledb_ctx; tiledb_ctx_init(&tiledb_ctx, NULL); // Array name const char* array_name = "my_workspace/sparse_arrays/my_array_B"; // Prepare cell buffers // --- First write --- int buffer_a1_1[] = { 7, 5, 0 }; size_t buffer_a2_1[] = { 0, 4, 6 }; const char buffer_var_a2_1[] = "hhhhffa"; float buffer_a3_1[] = { 7.1, 7.2, 5.1, 5.2, 0.1, 0.2 }; int64_t buffer_coords_1[] = { 3, 4, 4, 2, 1, 1 }; const void* buffers_1[] = { buffer_a1_1, buffer_a2_1, buffer_var_a2_1, buffer_a3_1, buffer_coords_1 }; size_t buffer_sizes_1[] = { sizeof(buffer_a1_1), sizeof(buffer_a2_1), sizeof(buffer_var_a2_1)-1, // No need to store the last '\0' character sizeof(buffer_a3_1), sizeof(buffer_coords_1) }; // --- Second write --- int buffer_a1_2[] = { 6, 4, 3, 1, 2 }; size_t buffer_a2_2[] = { 0, 3, 4, 8, 10 }; const char buffer_var_a2_2[] = "gggeddddbbccc"; float buffer_a3_2[] = { 6.1, 6.2, 4.1, 4.2, 3.1, 3.2, 1.1, 1.2, 2.1, 2.2 }; int64_t buffer_coords_2[] = { 3, 3, 3, 1, 2, 3, 1, 2, 1, 4 }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2, buffer_coords_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2), sizeof(buffer_a2_2), sizeof(buffer_var_a2_2)-1, // No need to store the last '\0' character sizeof(buffer_a3_2), sizeof(buffer_coords_2) }; #pragma omp parallel for // Write in parallel for(int i=0; i<2; ++i) { // Populate thread data const void** buffers; const size_t* buffer_sizes; if(i==0) { // First tile buffers = buffers_1; buffer_sizes = buffer_sizes_1; } else if(i==1) { // Second tile buffers = buffers_2; buffer_sizes = buffer_sizes_2; } // Write parallel_write( tiledb_ctx, array_name, buffers, buffer_sizes); } // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void parallel_write( const TileDB_CTX* tiledb_ctx, const char* array_name, const void** buffers, const size_t* buffer_sizes) { // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object array_name, // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Inapplicable NULL, // All attributes 0); // Number of attributes // Write to array tiledb_array_write(tiledb_array, buffers, buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); } #else #include int main() { printf("OpenMP not supported."); return 0; } #endif genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_primitive.cc000066400000000000000000000073721453617025200260230ustar00rootroot00000000000000/** * @file tiledb_array_primitive.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to initialize/finalize an array, and explore its schema. */ #include "examples.h" // Prints some schema info (you can enhance this to print the entire schema) void print_some_array_schema_info(const TileDB_ArraySchema* array_schema); int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // ----- Dense array ----- // // Load array schema when the array is not initialized TileDB_ArraySchema array_schema; CHECK_RC(tiledb_array_load_schema( tiledb_ctx, // Context "my_workspace/dense_arrays/my_array_A", // Array name &array_schema)); // Array schema struct // Print some array schema info print_some_array_schema_info(&array_schema); // Free array schema CHECK_RC(tiledb_array_free_schema(&array_schema)); // ----- Sparse array ----- // // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode NULL, // Subarray (whole domain) NULL, // Attributes (all attributes) 0)); // Number of attributes // Get array schema when the array is initialized CHECK_RC(tiledb_array_get_schema(tiledb_array, &array_schema)); // Print some schema info print_some_array_schema_info(&array_schema); // Free array schema CHECK_RC(tiledb_array_free_schema(&array_schema)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } void print_some_array_schema_info(const TileDB_ArraySchema* array_schema) { printf("Array name: %s\n", array_schema->array_name_); printf("Attributes: "); for(int i=0; iattribute_num_; ++i) printf("%s ", array_schema->attributes_[i]); printf("\n"); if(array_schema->dense_) printf("The array is dense\n"); else printf("The array is sparse\n"); } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_read_dense_1.cc000066400000000000000000000070071453617025200263170ustar00rootroot00000000000000/** * @file tiledb_array_read_dense_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read a complete dense array. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_READ, // Mode NULL, // Whole domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[16]; size_t buffer_a2[16]; char buffer_var_a2[40]; float buffer_a3[32]; void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2), sizeof(buffer_a3) }; // Read from array CHECK_RC(tiledb_array_read(tiledb_array, buffers, buffer_sizes)); // Print only non-empty cell values int64_t result_num = buffer_sizes[0] / sizeof(int); printf(" a1\t a2\t (a3.first, a3.second)\n"); printf("-----------------------------------------\n"); for(int i=0; i 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1)); // Number of attributes // Prepare cell buffers int buffer_a1[3]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Loop until no overflow printf(" a1\n----\n"); do { printf("Reading cells...\n"); // Read from array CHECK_RC(tiledb_array_read(tiledb_array, buffers, buffer_sizes)); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); for(int i=0; i int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subarray and attributes int64_t subarray[] = { 3, 3, 2, 2 }; int64_t subarray_2[] = { 4, 4, 3, 3 }; const char* attributes[] = { "a1" }; const char* attributes_2[] = { "a2" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1); // Number of attributes // Prepare cell buffers int buffer_a1[1]; size_t buffer_a2[1]; char buffer_var_a2[10]; void* buffers[] = { buffer_a1 }; void* buffers_2[] = { buffer_a2, buffer_var_a2 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; size_t buffer_sizes_2[] = { sizeof(buffer_a2), sizeof(buffer_var_a2) }; // Read from array - #1 tiledb_array_read(tiledb_array, buffers, buffer_sizes); printf("a1 for (3,2): %3d\n", buffer_a1[0]); // Reset subarray and attributes tiledb_array_reset_subarray(tiledb_array, subarray_2); tiledb_array_reset_attributes(tiledb_array, attributes_2, 1); // Read from array - #2 tiledb_array_read(tiledb_array, buffers_2, buffer_sizes_2); printf("a2 for (4,3): %3.*s\n", int(buffer_sizes_2[1]), buffer_var_a2); // Finalize the array tiledb_array_finalize(tiledb_array); /* Finalize context. */ tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_read_dense_sorted.cc000066400000000000000000000063121453617025200274550ustar00rootroot00000000000000/** * @file tiledb_array_read_dense_sorted.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from a dense array, constraining the read * to a specific subarray and subset of attributes. The cells are copied to the * input buffers sorted in row-major order within the selected subarray. */ #include "tiledb.h" #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_READ_SORTED_ROW, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1); // Number of attributes // Prepare cell buffers int buffer_a1[3]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Loop until no overflow printf(" a1\n----\n"); do { // Read from array printf("Reading cells...\n"); tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); for(int i=0; i #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode NULL, // Whole domain NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[10]; size_t buffer_a2[10]; char buffer_var_a2[30]; float buffer_a3[20]; int64_t buffer_coords[20]; void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2), sizeof(buffer_a3), sizeof(buffer_coords) }; // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); printf("coords\t a1\t a2\t (a3.first, a3.second)\n"); printf("--------------------------------------------------\n"); for(int i=0; i int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1); // Number of attributes // Prepare cell buffers int buffer_a1[2]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Loop until no overflow printf(" a1\n----\n"); do { printf("Reading cells...\n"); // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); for(int i=0; i #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode NULL, // Whole domain NULL, // All attributes 0); // Number of attributes tiledb_array_apply_filter(tiledb_array, "a1 > 5 && (a2 == \"u\" || a2 != \"w\") && a3[0] > 105.0 && a2 != \"yyy\""); // Prepare cell buffers int buffer_a1[10]; size_t buffer_a2[10]; char buffer_var_a2[30]; float buffer_a3[20]; int64_t buffer_coords[20]; void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2), sizeof(buffer_a3), sizeof(buffer_coords) }; // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); printf("coords\t a1\t a2\t (a3.first, a3.second)\n"); printf("--------------------------------------------------\n"); int64_t positions[10]; for(int i=0; i int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1); // Number of attributes tiledb_array_apply_filter(tiledb_array, "a1 > 5 && a1 != 104"); // Prepare cell buffers int buffer_a1[2]; void* buffers[] = { buffer_a1 }; // Loop until no overflow printf(" a1\n----\n"); do { printf("Reading cells...\n"); size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); int64_t positions[1]; for(int i=0; i int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subarray and attributes int64_t subarray[] = { 3, 4, 2, 4 }; const char* attributes[] = { "a1" }; // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_READ_SORTED_ROW, // Mode subarray, // Constrain in subarray attributes, // Subset on attributes 1); // Number of attributes // Prepare cell buffers int buffer_a1[2]; void* buffers[] = { buffer_a1 }; size_t buffer_sizes[] = { sizeof(buffer_a1) }; // Loop until no overflow printf(" a1\n----\n"); do { printf("Reading cells...\n"); // Read from array tiledb_array_read(tiledb_array, buffers, buffer_sizes); // Print cell values int64_t result_num = buffer_sizes[0] / sizeof(int); for(int i=0; i 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } int64_t subarray[] = { 3, 4, 3, 4 }; // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_WRITE, // Mode subarray, // Subarray NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 112, 113, 114, 115 }; size_t buffer_a2[] = { 0, 1, 3, 6 }; const char buffer_var_a2[] = "MNNOOOPPPP"; float buffer_a3[] = { 112.1, 112.2, 113.1, 113.2, 114.1, 114.2, 115.1, 115.2 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3) }; // Write to array CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_update_dense_2.cc000066400000000000000000000060501453617025200266640ustar00rootroot00000000000000/** * @file tiledb_array_update_dense_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to update a dense array, writing random sparse updates. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Whole domain (ignored) NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 211, 213, 212, 208 }; size_t buffer_a2[] = { 0, 4, 6, 7 }; const char buffer_var_a2[] = "wwwwyyxu"; float buffer_a3[] = { 211.1, 211.2, 213.1, 213.2, 212.1, 212.2, 208.1, 208.2 }; int64_t buffer_coords[] = { 4, 2, 3, 4, 3, 3, 3, 1 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array tiledb_array_write(tiledb_array, buffers, buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_update_sparse_1.cc000066400000000000000000000061561453617025200270710ustar00rootroot00000000000000/** * @file tiledb_array_update_sparse_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to update a sparse array. Observe that this is simply * a write operation. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Entire domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 107, 104, 106, 105 }; size_t buffer_a2[] = { 0, 3, 4, 5 }; const char buffer_var_a2[] = "yyyuwvvvv"; float buffer_a3[] = { 107.1, 107.2, 104.1, 104.2, 106.1, 106.2, 105.1, 105.2 }; int64_t buffer_coords[] = { 3, 4, 3, 2, 3, 3, 4, 1 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_update_sparse_2.cc000066400000000000000000000062251453617025200270670ustar00rootroot00000000000000/** * @file tiledb_array_update_sparse_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to update a sparse array, including how to handle * deletions. Observe that this is simply a write operation. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Entire domain NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 107, TILEDB_EMPTY_INT32, 106, 105 }; size_t buffer_a2[] = { 0, 3, 4, 5 }; const char buffer_var_a2[] = { 'y', 'y', 'y', TILEDB_EMPTY_CHAR, 'w', 'v', 'v', 'v', 'v' }; float buffer_a3[] = { 107.1, 107.2, TILEDB_EMPTY_FLOAT32, TILEDB_EMPTY_FLOAT32, 106.1, 106.2, 105.1, 105.2 }; int64_t buffer_coords[] = { 3, 4, 3, 2, 3, 3, 4, 1 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2), sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array tiledb_array_write(tiledb_array, buffers, buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_dense_1.cc000066400000000000000000000100751453617025200265350ustar00rootroot00000000000000/** * @file tiledb_array_write_dense_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a dense array. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_WRITE, // Mode NULL, // Entire domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 0, 1, 2, 3, // Upper left tile 4, 5, 6, 7, // Upper right tile 8, 9, 10, 11, // Lower left tile 12, 13, 14, 15 // Lower right tile }; size_t buffer_a2[] = { 0, 1, 3, 6, // Upper left tile 10, 11, 13, 16, // Upper right tile 20, 21, 23, 26, // Lower left tile 30, 31, 33, 36 // Lower right tile }; const char buffer_var_a2[] = "abbcccdddd" // Upper left tile "effggghhhh" // Upper right tile "ijjkkkllll" // Lower left tile "mnnooopppp"; // Lower right tile float buffer_a3[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2, // Upper left tile 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2, // Upper right tile 8.1, 8.2, 9.1, 9.2, 10.1, 10.2, 11.1, 11.2, // Lower left tile 12.1, 12.2, 13.1, 13.2, 14.1, 14.2, 15.1, 15.2, // Lower right tile }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3) }; // Write to array CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_dense_2.cc000066400000000000000000000077411453617025200265440ustar00rootroot00000000000000/** * @file tiledb_array_write_dense_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a dense array invoking the write function * multiple times. This will have the same effect as program * tiledb_array_write_dense_1.cc. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_WRITE, // Mode NULL, // Entire domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers - #1 int buffer_a1[] = { 0, 1, 2, 3, 4, 5 }; size_t buffer_a2[] = { 0, 1, 3, 6, 10, 11, 13, 16 }; const char buffer_var_a2[] = "abbcccddddeffggghhhh"; float* buffer_a3 = NULL; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { 6*sizeof(int), // 6 cels on a1 8*sizeof(size_t), 20, // 8 cells on a2 0 // no cells on a3 }; // Write to array - #1 CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Prepare cell buffers - #2 int buffer_a1_2[] = { 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; size_t buffer_a2_2[] = { 0, 1, 3, 6, 10, 11, 13, 16 }; const char buffer_var_a2_2[] = "ijjkkkllllmnnooopppp"; float buffer_a3_2[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2, // Upper left tile 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2, // Upper right tile 8.1, 8.2, 9.1, 9.2, 10.1, 10.2, 11.1, 11.2, // Lower left tile 12.1, 12.2, 13.1, 13.2, 14.1, 14.2, 15.1, 15.2, // Lower right tile }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2 }; size_t buffer_sizes_2[] = { 10*sizeof(int), // 6 cels on a1 8*sizeof(size_t), 20, // 8 cells on a2 32*sizeof(float) // 16 cells on a3 (2 values each) }; // Write to array - #2 CHECK_RC(tiledb_array_write(tiledb_array, buffers_2, buffer_sizes_2)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_dense_sorted.cc000066400000000000000000000064511453617025200277000ustar00rootroot00000000000000/** * @file tiledb_array_write_dense_sorted.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a dense array, providing the array cells sorted * in row-major order within the specified subarray. TileDB will properly * re-organize the cells into the global cell order, prior to writing them * on the disk. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Set the subarray where the write will focus on int64_t subarray[] = { 3, 4, 2, 4 }; // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/dense_arrays/my_array_A", // Array name TILEDB_ARRAY_WRITE_SORTED_ROW, // Mode subarray, // Subarray NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 9, 12, 13, 11, 14, 15 }; size_t buffer_a2[] = { 0, 2, 3, 5, 9, 12 }; const char buffer_var_a2[] = "jjmnnllllooopppp"; float buffer_a3[] = { 9.1, 9.2, 12.1, 12.2, 13.1, 13.2, 11.1, 11.2, 14.1, 14.2, 15.1, 15.2 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3) }; // Write to array CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_sparse_1.cc000066400000000000000000000062421453617025200267350ustar00rootroot00000000000000/** * @file tiledb_array_write_sparse_1.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a sparse array. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_WRITE, // Mode NULL, // Entire domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 0, 1, 2, 3, 4, 5, 6, 7 }; size_t buffer_a2[] = { 0, 1, 3, 6, 10, 11, 13, 16 }; const char buffer_var_a2[] = "abbcccddddeffggghhhh"; float buffer_a3[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2 }; int64_t buffer_coords[] = { 1, 1, 1, 2, 1, 4, 2, 3, 3, 1, 4, 2, 3, 3, 3, 4 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_sparse_2.cc000066400000000000000000000073731453617025200267440ustar00rootroot00000000000000/** * @file tiledb_array_write_sparse_2.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write to a sparse array with two sorted batch writes. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Initialize array TileDB_Array* tiledb_array; CHECK_RC(tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_WRITE, // Mode NULL, // Entire domain NULL, // All attributes 0)); // Number of attributes // Prepare cell buffers - #1 int buffer_a1[] = { 0, 1, 2 }; size_t buffer_a2[] = { 0, 1, 3, 6, 10, 11, 13, 16 }; const char buffer_var_a2[] = "abbcccddddeffggghhhh"; float buffer_a3[] = { 0.1, 0.2, 1.1, 1.2, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2, 5.1, 5.2, 6.1, 6.2, 7.1, 7.2 }; int64_t buffer_coords[] = { 1, 1, 1, 2 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array - #1 CHECK_RC(tiledb_array_write(tiledb_array, buffers, buffer_sizes)); // Prepare cell buffers - #2 int buffer_a1_2[] = { 3, 4, 5, 6, 7 }; size_t* buffer_a2_2 = NULL; const char* buffer_var_a2_2 = NULL; float* buffer_a3_2 = NULL; int64_t buffer_coords_2[] = { 1, 4, 2, 3, 3, 1, 4, 2, 3, 3, 3, 4 }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2, buffer_coords_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2), 0, 0, 0, sizeof(buffer_coords_2) }; // Write to array - #2 CHECK_RC(tiledb_array_write(tiledb_array, buffers_2, buffer_sizes_2)); // Finalize array CHECK_RC(tiledb_array_finalize(tiledb_array)); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_sparse_3.cc000066400000000000000000000061651453617025200267430ustar00rootroot00000000000000/** * @file tiledb_array_write_sparse_3.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write unsorted cells to a sparse array. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Entire domain NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 7, 5, 0, 6, 4, 3, 1, 2 }; size_t buffer_a2[] = { 0, 4, 6, 7, 10, 11, 15, 17 }; const char buffer_var_a2[] = "hhhhffagggeddddbbccc"; float buffer_a3[] = { 7.1, 7.2, 5.1, 5.2, 0.1, 0.2, 6.1, 6.2, 4.1, 4.2, 3.1, 3.2, 1.1, 1.2, 2.1, 2.2 }; int64_t buffer_coords[] = { 3, 4, 4, 2, 1, 1, 3, 3, 3, 1, 2, 3, 1, 2, 1, 4 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array tiledb_array_write(tiledb_array, buffers, buffer_sizes); // Finalize array tiledb_array_finalize(tiledb_array); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_array_write_sparse_4.cc000066400000000000000000000074241453617025200267430ustar00rootroot00000000000000/** * @file tiledb_array_write_sparse_4.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write unsorted cells to a sparse array with two batch writes. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize array TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, // Context &tiledb_array, // Array object "my_workspace/sparse_arrays/my_array_B", // Array name TILEDB_ARRAY_WRITE_UNSORTED, // Mode NULL, // Entire domain NULL, // All attributes 0); // Number of attributes // Prepare cell buffers - #1 int buffer_a1[] = { 7, 5, 0 }; size_t buffer_a2[] = { 0, 4, 6 }; const char buffer_var_a2[] = "hhhhffa"; float buffer_a3[] = { 7.1, 7.2, 5.1, 5.2, 0.1, 0.2 }; int64_t buffer_coords[] = { 3, 4, 4, 2, 1, 1 }; const void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3, buffer_coords }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2), sizeof(buffer_var_a2)-1, // No need to store the last '\0' character sizeof(buffer_a3), sizeof(buffer_coords) }; // Write to array - #1 tiledb_array_write(tiledb_array, buffers, buffer_sizes); // Prepare cell buffers - #2 int buffer_a1_2[] = { 6, 4, 3, 1, 2 }; size_t buffer_a2_2[] = { 0, 3, 4, 8, 10 }; const char buffer_var_a2_2[] = "gggeddddbbccc"; float buffer_a3_2[] = { 6.1, 6.2, 4.1, 4.2, 3.1, 3.2, 1.1, 1.2, 2.1, 2.2 }; int64_t buffer_coords_2[] = { 3, 3, 3, 1, 2, 3, 1, 2, 1, 4 }; const void* buffers_2[] = { buffer_a1_2, buffer_a2_2, buffer_var_a2_2, buffer_a3_2, buffer_coords_2 }; size_t buffer_sizes_2[] = { sizeof(buffer_a1_2), sizeof(buffer_a2_2), sizeof(buffer_var_a2_2)-1, // No need to store the last '\0' character sizeof(buffer_a3_2), sizeof(buffer_coords_2) }; // Write to array - #2 tiledb_array_write(tiledb_array, buffers_2, buffer_sizes_2); // Finalize array tiledb_array_finalize(tiledb_array); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_catching_errors.cc000066400000000000000000000043701453617025200257640ustar00rootroot00000000000000/** * @file tiledb_catching_errors.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * This examples shows how to catch errors. */ #include "tiledb.h" #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Create a workspace int rc = tiledb_workspace_create(tiledb_ctx, "my_workspace"); if(rc == TILEDB_OK) printf("Workspace created successfully!\n"); else if(rc == TILEDB_ERR) printf("%s\n", tiledb_errmsg); // Create the same workspace again - ERROR rc = tiledb_workspace_create(tiledb_ctx, "my_workspace"); if(rc == TILEDB_OK) printf("Workspace created successfully!\n"); else if(rc == TILEDB_ERR) printf("%s\n", tiledb_errmsg); // Print the TileDB error message // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_clear_delete_move.cc000066400000000000000000000040021453617025200262360ustar00rootroot00000000000000/** * @file tiledb_clear_delete_move.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to clear, delete and move TileDB objects. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Clear an array tiledb_clear(tiledb_ctx, "my_workspace/sparse_arrays/my_array_B"); // Delete a group tiledb_delete(tiledb_ctx, "my_workspace/dense_arrays"); // Move a workspace tiledb_move(tiledb_ctx, "my_workspace", "my_workspace_2"); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_config.cc000066400000000000000000000040211453617025200240460ustar00rootroot00000000000000/** * @file tiledb_config.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to set the TileDB configuration parameters. */ #include "tiledb.h" #include int main() { /* Create a TileDB configuration. */ TileDB_Config tiledb_config = {}; /* * IMPORTANT: You need to zero out the members of the config structure if you * are setting only a subset of them, so that the rest can take default * values. */ tiledb_config.home_ = "."; // TileDB home will be the current directory tiledb_config.read_method_ = TILEDB_IO_READ; // OS read instead of mmap // Initialize context with the default configuration parameters TileDB_CTX* tiledb_ctx; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); /* --- Your code here --- */ /* Finalize context. */ tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_delete.cc000066400000000000000000000042551453617025200240540ustar00rootroot00000000000000/** * @file tiledb_delete.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2019 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Recursive delete a TileDB directory */ #include "tiledb.h" #include "tiledb_utils.h" #include #include int main(int argc, char *argv[]) { if (argc > 2 && strcmp(argv[2], "-f") == 0) { std::cerr << "Forcibly deleting everything under " << argv[1] << std::endl; if (TileDBUtils::is_dir(argv[1])) { TileDBUtils::delete_dir(argv[1]); } return 0; } // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); // Delete a workspace/group/array/fragment tiledb_delete(tiledb_ctx, argv[1]); // Finalize context tiledb_ctx_finalize(tiledb_ctx); } else { std::cerr << "Usage: tiledb_delete [-f]" << std::endl; } return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_ls.cc000066400000000000000000000062061453617025200232260ustar00rootroot00000000000000/** * @file tiledb_list.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to explore the contents of a TileDB directory. */ #include "examples.h" int main(int argc, char** argv) { char *home; TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; home = strdup(argv[1]); tiledb_config.home_ = home; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { home = (char *)malloc(TILEDB_NAME_MAX_LEN+1); home = getcwd(home, TILEDB_NAME_MAX_LEN); CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Retrieve number of directories int dir_num; CHECK_RC(tiledb_ls_c(tiledb_ctx, home, &dir_num)); // Exit if there are not TileDB objects in the input directory_ if(dir_num == 0) return 0; // Initialize variables char** dirs = new char*[dir_num]; int* dir_types = new int[dir_num]; for(int i=0; i int main(int argc, char *argv[]) { char *home; TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; home = strdup(argv[1]); tiledb_config.home_ = home; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { home = (char *)malloc(TILEDB_NAME_MAX_LEN+1); home = getcwd(home, TILEDB_NAME_MAX_LEN); CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Retrieve number of workspaces int workspace_num; CHECK_RC(tiledb_ls_workspaces_c(tiledb_ctx, home, &workspace_num)); // Exit if there are no workspaces if(workspace_num == 0) return 0; // Allocate buffers char** workspaces = new char*[workspace_num]; for(int i=0; i 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Consolidate metadata tiledb_metadata_consolidate( tiledb_ctx, "my_workspace/sparse_arrays/my_array_B/meta"); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_metadata_create.cc000066400000000000000000000060641453617025200257150ustar00rootroot00000000000000/** * @file tiledb_metadata_create.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It creates a metadata object. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Prepare parameters for metadata schema const char* metadata_name = "my_workspace/sparse_arrays/my_array_B/meta"; const char* attributes[] = { "a1", "a2" }; // Two attributes const int cell_val_num[] = { 1, // a1 TILEDB_VAR_NUM // a2 }; const int compression[] = { TILEDB_GZIP, // a1 TILEDB_GZIP, // a2 TILEDB_NO_COMPRESSION // TILEDB_KEY }; const int types[] = { TILEDB_INT32, // a1 TILEDB_CHAR // a2 }; // Set metadata schema TileDB_MetadataSchema metadata_schema; tiledb_metadata_set_schema( &metadata_schema, // Metadata schema struct metadata_name, // Metadata name attributes, // Attributes 2, // Number of attributes 4, // Capacity cell_val_num, // Number of cell values per attribute compression, // Compression NULL, // Compression level - use defaults types // Types ); // Create metadata tiledb_metadata_create(tiledb_ctx, &metadata_schema); // Free metadata schema tiledb_metadata_free_schema(&metadata_schema); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_metadata_iterator.cc000066400000000000000000000066051453617025200263040ustar00rootroot00000000000000/** * @file tiledb_metadata_iterator.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to use a metadata iterator. */ #include "tiledb.h" #include int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subset over the attributes const char* attributes[] = { TILEDB_KEY }; // Prepare cell buffers size_t buffer_key[8]; char buffer_key_var[500]; void* buffers[] = { buffer_key, buffer_key_var }; size_t buffer_sizes[] = { sizeof(buffer_key), sizeof(buffer_key_var) }; // Initialize metadata iterator TileDB_MetadataIterator* tiledb_metadata_iterator; tiledb_metadata_iterator_init( tiledb_ctx, // Context &tiledb_metadata_iterator, // Metadata iterator "my_workspace/sparse_arrays/my_array_B/meta", // Metadata name attributes, // Attributes 1, // Number of attributes buffers, // Buffers for internal use buffer_sizes); // Sizes of buffers // Iterate over the metadata const char* key; size_t key_size; while(!tiledb_metadata_iterator_end(tiledb_metadata_iterator)) { // Get value tiledb_metadata_iterator_get_value( tiledb_metadata_iterator, // Metadata iterator 0, // Attribute id (const void**) &key, // Key &key_size); // Key size // Print only if it is not empty if(key[0] != TILEDB_EMPTY_CHAR) printf("%s\n", key); // Advance iterator tiledb_metadata_iterator_next(tiledb_metadata_iterator); } // Finalize metadata iterator tiledb_metadata_iterator_finalize(tiledb_metadata_iterator); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_metadata_primitive.cc000066400000000000000000000073701453617025200264630ustar00rootroot00000000000000/** * @file tiledb_metadata_primitive.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to initialize/finalize a metadata object and explore its schema. */ #include "tiledb.h" #include // Prints some schema info (you can enhance this to print the entire schema) void print_some_metadata_schema_info( const TileDB_MetadataSchema* metadata_schema); int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // ----- Get schema without metadata initialization ----- // // Load metadata schema when the metadata object is not initialized TileDB_MetadataSchema metadata_schema; tiledb_metadata_load_schema( tiledb_ctx, // Context "my_workspace/sparse_arrays/my_array_B/meta", // Metadata name &metadata_schema); // Metadata schema struct // Print some metadata schema info print_some_metadata_schema_info(&metadata_schema); // Free metadata schema tiledb_metadata_free_schema(&metadata_schema); // ----- Get schema after metadata initialization ----- // // Initialize metadata TileDB_Metadata* tiledb_metadata; tiledb_metadata_init( tiledb_ctx, // Context &tiledb_metadata, // Array object "my_workspace/sparse_arrays/my_array_B/meta", // Array name TILEDB_METADATA_READ, // Mode NULL, // Attributes (all) 0); // Number of attributes // Get metadata schema when the metadata object is initialized tiledb_metadata_get_schema(tiledb_metadata, &metadata_schema); // Print some schema info print_some_metadata_schema_info(&metadata_schema); // Free metadata schema tiledb_metadata_free_schema(&metadata_schema); // Finalize metadata tiledb_metadata_finalize(tiledb_metadata); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } void print_some_metadata_schema_info( const TileDB_MetadataSchema* metadata_schema) { printf("Metadata name: %s\n", metadata_schema->metadata_name_); printf("Attributes: "); for(int i=0; iattribute_num_; ++i) printf("%s ", metadata_schema->attributes_[i]); printf("\n"); } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_metadata_read.cc000066400000000000000000000074301453617025200253630ustar00rootroot00000000000000/** * @file tiledb_metadata_read.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to read from metadata. */ #include "tiledb.h" #include int main(int argc, char** argv) { // Sanity check if(argc < 2) { fprintf(stderr, "Usage: ./tiledb_metadata_read key\n"); return -1; } // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 2) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[2]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Subset over attributes const char* attributes[] = { "a1", "a2" }; // Initialize metadata TileDB_Metadata* tiledb_metadata; tiledb_metadata_init( tiledb_ctx, // Context &tiledb_metadata, // Metadata object "my_workspace/sparse_arrays/my_array_B/meta", // Metadata name TILEDB_METADATA_READ, // Mode attributes, // Attributes 2); // Number of attributes // Prepare cell buffers int buffer_a1[1]; size_t buffer_a2[1]; char buffer_var_a2[2]; void* buffers[] = { buffer_a1, // a1 buffer_a2, buffer_var_a2 // a2 }; size_t buffer_sizes[] = { sizeof(buffer_a1), // a1 sizeof(buffer_a2), sizeof(buffer_var_a2) // a2 }; // Read from metadata tiledb_metadata_read(tiledb_metadata, argv[1], buffers, buffer_sizes); // Check existence if(buffer_sizes[0] == 0 && !tiledb_metadata_overflow(tiledb_metadata, 0)) { fprintf(stderr, "Key '%s' does not exist in the metadata!\n", argv[1]); } else if(buffer_sizes[2] == 0 && tiledb_metadata_overflow(tiledb_metadata, 1)) { // Check overflow for a2 fprintf(stderr, "Reading value on attribute 'a2' for key '%s' resulted in " "a buffer overflow!\n", argv[1]); } else if(static_cast(buffers[0])[0] == TILEDB_EMPTY_INT32) { // Check if deleted fprintf(stderr, "Key '%s' has been deleted!\n", argv[1]); } else { // Print attribute values printf( "%s: a1=%d, a2=%.*s\n", argv[1], static_cast(buffers[0])[0], int(buffer_sizes[2]), static_cast(buffers[2])); } /* Finalize the array. */ tiledb_metadata_finalize(tiledb_metadata); /* Finalize context. */ tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_metadata_update.cc000066400000000000000000000071141453617025200257310ustar00rootroot00000000000000/** * @file tiledb_metadata_update.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to modify or delete a metadata item. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize metadata TileDB_Metadata* tiledb_metadata; tiledb_metadata_init( tiledb_ctx, // Context &tiledb_metadata, // Metadata object "my_workspace/sparse_arrays/my_array_B/meta", // Metadata name TILEDB_METADATA_WRITE, // Mode NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 100, TILEDB_EMPTY_INT32 }; size_t buffer_a2[] = { 0, 1 }; char buffer_var_a2[] = { 'A', TILEDB_EMPTY_CHAR }; char keys[] = "k1\0k2"; size_t buffer_keys[] = { 0, 3 }; char buffer_var_keys[] = { 'k', '1', '\0', // k1 modified key value TILEDB_EMPTY_CHAR // k2 deleted value }; const void* buffers[] = { buffer_a1, // a1 buffer_a2, buffer_var_a2, // a2 buffer_keys, buffer_var_keys // TILEDB_KEY }; size_t buffer_sizes[] = { sizeof(buffer_a1), // a1 sizeof(buffer_a2), sizeof(buffer_var_a2), // a2 sizeof(buffer_keys), sizeof(buffer_var_keys) // TILEDB_KEY }; size_t keys_size = sizeof(keys); // Write metadata tiledb_metadata_write( tiledb_metadata, // Metadata object keys, // Keys keys_size, // Keys size buffers, // Attribute buffers buffer_sizes); // Attribute buffer sizes // Finalize the metadata tiledb_metadata_finalize(tiledb_metadata); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_metadata_write.cc000066400000000000000000000065731453617025200256110ustar00rootroot00000000000000/** * @file tiledb_metadata_write.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It shows how to write metadata. */ #include "tiledb.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; tiledb_ctx_init(&tiledb_ctx, &tiledb_config); } else { tiledb_ctx_init(&tiledb_ctx, NULL); } // Initialize metadata TileDB_Metadata* tiledb_metadata; tiledb_metadata_init( tiledb_ctx, // Context &tiledb_metadata, // Metadata object "my_workspace/sparse_arrays/my_array_B/meta", // Metadata name TILEDB_METADATA_WRITE, // Mode NULL, // All attributes 0); // Number of attributes // Prepare cell buffers int buffer_a1[] = { 1, 2, 3 }; size_t buffer_a2[] = { 0, 1, 3 }; char buffer_var_a2[] = "abbccc"; size_t buffer_keys[] = { 0, 3, 6 }; char buffer_var_keys[] = "k1\0k2\0k3"; const void* buffers[] = { buffer_a1, // a1 buffer_a2, buffer_var_a2, // a2 buffer_keys, buffer_var_keys // TILEDB_KEY }; size_t buffer_sizes[] = { sizeof(buffer_a1), // a1 sizeof(buffer_a2), sizeof(buffer_var_a2), // a2 sizeof(buffer_keys), sizeof(buffer_var_keys) // TILEDB_KEY }; size_t keys_size = sizeof(buffer_var_keys); // Write metadata tiledb_metadata_write( tiledb_metadata, // Metadata object buffer_var_keys, // Keys keys_size, // Keys size buffers, // Attribute buffers buffer_sizes); // Attribute buffer sizes // Finalize the metadata tiledb_metadata_finalize(tiledb_metadata); // Finalize context tiledb_ctx_finalize(tiledb_ctx); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/src/tiledb_workspace_group_create.cc000066400000000000000000000043471453617025200273510ustar00rootroot00000000000000/** * @file tiledb_workspace_group_create.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * It creates a workspace and two groups. */ #include "examples.h" int main(int argc, char *argv[]) { // Initialize context with home dir if specified in command line, else // initialize with the default configuration parameters TileDB_CTX* tiledb_ctx; if (argc > 1) { TileDB_Config tiledb_config; tiledb_config.home_ = argv[1]; CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); } else { CHECK_RC(tiledb_ctx_init(&tiledb_ctx, NULL)); } // Delete a workspace if it exists if (is_dir(tiledb_ctx, "my_workspace")) { CHECK_RC(tiledb_delete(tiledb_ctx, "my_workspace")); } // Create a workspace CHECK_RC(tiledb_workspace_create(tiledb_ctx, "my_workspace")); // Create a group in the worskpace CHECK_RC(tiledb_group_create(tiledb_ctx, "my_workspace/dense_arrays")); // Create two groups in the worskpace CHECK_RC(tiledb_group_create(tiledb_ctx, "my_workspace/sparse_arrays")); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/examples/tiledb.supp000066400000000000000000000026551453617025200223470ustar00rootroot00000000000000{ Memcheck:Leak match-leak-kinds: reachable fun:malloc obj:/usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0 obj:/usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0 obj:/usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0 fun:call_init.part.0 fun:call_init fun:_dl_init obj:/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 obj:* obj:* obj:* } { Memcheck:Leak match-leak-kinds: reachable fun:malloc obj:/usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0 obj:/usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0 obj:/usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0 fun:call_init.part.0 fun:call_init fun:_dl_init obj:/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN3Aws31InitializeEnumOverflowContainerEv fun:_ZN3Aws7InitAPIERKNS_10SDKOptionsE fun:_Z15awssdk_init_apiv fun:_ZSt13__invoke_implIvRFvvEJEET_St14__invoke_otherOT0_DpOT1_ fun:_ZSt8__invokeIRFvvEJEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ fun:_ZZSt9call_onceIRFvvEJEEvRSt9once_flagOT_DpOT0_ENKUlvE_clEv fun:_ZZNSt9once_flag18_Prepare_executionC4IZSt9call_onceIRFvvEJEEvRS_OT_DpOT0_EUlvE_EERS6_ENKUlvE_clEv fun:_ZZNSt9once_flag18_Prepare_executionC4IZSt9call_onceIRFvvEJEEvRS_OT_DpOT0_EUlvE_EERS6_ENUlvE_4_FUNEv fun:__pthread_once_slow fun:_ZL14__gthread_oncePiPFvvE fun:_ZSt9call_onceIRFvvEJEEvRSt9once_flagOT_DpOT0_ } genomicsdb-0.0~git20231212.9d7ddd0/test/000077500000000000000000000000001453617025200173245ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/CMakeLists.txt000066400000000000000000000044561453617025200220750ustar00rootroot00000000000000# # test/CMakeLists.txt # # # The MIT License # # Copyright (c) 2016 MIT and Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # Include test header directories file(GLOB TILEDB_TEST_INCLUDE_DIRS "include/*") include_directories(${TILEDB_TEST_INCLUDE_DIRS}) # Include TileDB core header directories file(GLOB TILEDB_CORE_INCLUDE_DIRS "${CMAKE_CURRENT_LIST_DIR}/../core/include/*") include_directories(${TILEDB_CORE_INCLUDE_DIRS}) # Get test source files file(GLOB_RECURSE TILEDB_TEST_SOURCES "src/*/test*.cc") # Add TILEDB_TEST_DIR macro add_definitions(-DTILEDB_TEST_DIR="${CMAKE_CURRENT_SOURCE_DIR}") # Catch2 major versions are both supported, so find Catch2 and set version include(SetCatch2Version) # Build one executable per test foreach(TILEDB_TEST_SOURCE ${TILEDB_TEST_SOURCES}) get_filename_component(FILENAME ${TILEDB_TEST_SOURCE} NAME) string(REGEX REPLACE ".cc$" "" TARGET ${FILENAME}) add_executable(${TARGET} EXCLUDE_FROM_ALL ${TILEDB_TEST_SOURCE}) target_link_libraries(${TARGET} tiledb_static ${TILEDB_LIB_DEPENDENCIES} Catch2::Catch2) add_test(${TARGET} ${TARGET}) set(ALL_CMDS ${ALL_CMDS} ${TARGET}) endforeach() # Add custom target 'check' add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} -V DEPENDS ${ALL_CMDS}) # Add check to all tests add_dependencies(tests ${ALL_CMDS}) genomicsdb-0.0~git20231212.9d7ddd0/test/include/000077500000000000000000000000001453617025200207475ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/include/benchmark/000077500000000000000000000000001453617025200227015ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/include/benchmark/tiledb_benchmark.h000066400000000000000000000500241453617025200263300ustar00rootroot00000000000000/** * @file tiledb_benchmark.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Benchmarking Configuration */ #ifndef TILEDB_TESTS_H #define TILEDB_TESTS_H #include "array_schema.h" #include "tiledb.h" #include "tiledb_constants.h" #include "tiledb_storage.h" #include "tiledb_utils.h" #include "utils.h" #include #include #include class BenchmarkConfig: public TempDir { public: BenchmarkConfig() { do_benchmark_ = is_env_set("TILEDB_BENCHMARK"); if (!do_benchmark_) { return; } std::ifstream config_stream(g_benchmark_config); if (config_stream.is_open()) { std::string line; while(getline(config_stream, line)) { line.erase(std::remove_if(line.begin(), line.end(), isspace), line.end()); if(line[0] != '#' && !line.empty()) { auto found = line.find("="); if (found != std::string::npos) { auto name = line.substr(0, found); auto value = line.substr(found + 1); parse(name, value); } } } CHECK(attributes_.size() == attribute_cell_val_nums_.size()); CHECK(attributes_.size() == attribute_data_types_.size()); // Add data and compression for coords attribute_data_types_.push_back(TILEDB_INT64); if (compression_.size() == attributes_.size()) { compression_.push_back(TILEDB_NO_COMPRESSION); } } else { throw std::invalid_argument("Could not open benchmark config " + g_benchmark_config + " for reading"); } } template void* create_buffer(int64_t size, T** buffer, size_t* bytes, bool is_write, bool is_offsets=false) { T *typed_buffer= *buffer; typed_buffer = new T[size]; *bytes = (size)*sizeof(T); for(int64_t i = 0; is_write && i < size; ++i) { if (is_offsets) { typed_buffer[i] = (T)i; } else { // TODO: Make this a template function and configurable via benchmark.config later // typed_buffer[i] = (T)(i<<2); // typed_buffer[i] = (T)(i%2); // typed_buffer[i] = (T)((i%2)-1); // typed_buffer[i] = (T)(i%3); // typed_buffer[i] = (T)((i%3)-1); // typed_buffer[i] = (T)(rand()%2); // typed_buffer[i] = (T)(rand()%3); typed_buffer[i] = (T)((rand()%3)-1); } } return reinterpret_cast(typed_buffer); } void create_buffers(bool is_write) { buffers_.clear(); buffer_sizes_.clear(); size_t nbytes; int num_cells = is_write?num_cells_to_write_:num_cells_to_read_; for(auto i=0ul; i array_names_; int io_read_mode_ = TILEDB_IO_MMAP; int io_write_mode_ = TILEDB_IO_WRITE; int array_read_mode_ = TILEDB_ARRAY_READ; int array_write_mode_ = TILEDB_ARRAY_WRITE; int fragments_per_array_ = 1; std::vector attribute_names_; std::vector attributes_; std::vector attribute_data_types_; std::vector attribute_cell_val_nums_; int num_var_attributes_ = 0; bool sparse_ = true; std::vector dimension_names_; std::vector dimensions_; std::vector domain_; std::vector compression_; std::vector compression_level_; std::vector offsets_compression_; std::vector offsets_compression_level_; int num_cells_per_tile_ = 0; int num_cells_to_write_ = 1024; int num_cells_to_read_ = 1024; bool human_readable_sizes_ = true; bool print_array_schema_ = true; std::vector buffers_; std::vector buffer_sizes_; private: void parse(const std::string& name, const std::string& value) { if (!name.empty() && !value.empty()) { std::stringstream value_stream(value); std::string token; if (name == "Array_Names") { parse_array_names(value); } else if (name == "IO_Write_Mode") { io_write_mode_ = std::stoi(value); } else if (name == "IO_Read_Mode") { io_read_mode_ = std::stoi(value); } else if (name == "Array_Write_Mode") { array_write_mode_ = std::stoi(value); } else if (name == "Array_Read_Mode") { array_read_mode_ = std::stoi(value); } else if (name == "Fragments_Per_Array") { fragments_per_array_ = std::stoi(value); } else if (name == "Attribute_Names") { parse_names(attribute_names_, value); transform(attribute_names_, attributes_); } else if (name == "Attribute_Data_Types") { parse_attribute_data_types(value); } else if (name == "Attribute_Cell_Val_Num") { parse_attribute_cell_val_num(value); } else if (name == "Dimension_Names") { parse_names(dimension_names_, value); transform(dimension_names_, dimensions_); } else if (name == "Domain") { parse_domain(value); } else if (name == "Compression") { parse_compression(compression_, value); } else if (name == "Compression_Levels") { parse_compression(compression_level_, value); } else if (name == "Offsets_Compression") { parse_compression(offsets_compression_, value); } else if (name == "Offsets_Compression_Levels") { parse_compression(offsets_compression_level_, value); } else if (name == "Cells_Per_Tile") { num_cells_per_tile_ = std::stoi(value); } else if (name == "Cells_To_Write") { num_cells_to_write_ = std::stoi(value); } else if (name == "Cells_To_Read") { num_cells_to_read_ = std::stoi(value); } else if (name == "Print_Human_Readable_Sizes") { human_readable_sizes_ = (std::stoi(value) != 0); } else if (name == "Print_Array_Schema") { print_array_schema_ = (std::stoi(value) != 0); } else { std::cerr << "Unrecognized benchmark config name : " << name << std::endl; } } } void parse_names(std::vector& names, const std::string& value) { std::stringstream value_stream(value); std::string token; while (getline(value_stream, token, ',')) { names.push_back(token); } } void transform(std::vector& names, std::vector& char_names) { char_names.resize(names.size()); for(auto i=0u; i 0) { attribute_cell_val_nums_.push_back(cell_val_num); } else { attribute_cell_val_nums_.push_back(TILEDB_VAR_NUM); num_var_attributes_++; } } } void parse_domain(const std::string& value) { std::stringstream value_stream(value); std::string token; while (getline(value_stream, token, ',')) { domain_.push_back(std::stoll(token)); } if (dimensions_.size() > 0) { CHECK(domain_.size() == dimensions_.size()*2); } } void parse_compression(std::vector& compression, const std::string& value) { std::stringstream value_stream(value); std::string token; while (getline(value_stream, token, ',')) { compression.push_back(std::stoi(token)); } } }; // TileDB Benchmark Utils void create_workspace(BenchmarkConfig* config) { TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = config->get_temp_dir().c_str(); REQUIRE(tiledb_ctx_init(&tiledb_ctx, &tiledb_config) == TILEDB_OK); REQUIRE(tiledb_workspace_create(tiledb_ctx, config->workspace_.c_str()) == TILEDB_OK); std::cerr << "Workspace: " << config->workspace_ << "\n\n"; CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } void delete_arrays(BenchmarkConfig* config) { TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = config->get_temp_dir().c_str(); REQUIRE(tiledb_ctx_init(&tiledb_ctx, &tiledb_config) == TILEDB_OK); for(auto dir : get_dirs(tiledb_ctx, config->workspace_)) { std::cerr<<"deleting "<< dir<<"\n"; delete_dir(tiledb_ctx, dir); } CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } void create_arrays(BenchmarkConfig* config, int i) { TileDB_ArraySchema array_schema; REQUIRE(tiledb_array_set_schema(&array_schema, config->array_names_[i].c_str(), config->attributes_.data(), config->attributes_.size(), config->num_cells_per_tile_, TILEDB_COL_MAJOR, config->attribute_cell_val_nums_.data(), config->compression_.data(), config->compression_level_.data(), config->offsets_compression_.data(), config->offsets_compression_level_.data(), 0, //sparse array - hardcoded config->dimensions_.data(), config->dimensions_.size(), config->domain_.data(), config->dimensions_.size()*2*sizeof(int64_t), // size of domain NULL, // tile extents 0, // tile extents length in bytes 0, // tile order config->attribute_data_types_.data()) == TILEDB_OK); TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = config->get_temp_dir().c_str(); REQUIRE(tiledb_ctx_init(&tiledb_ctx, &tiledb_config) == TILEDB_OK); REQUIRE(tiledb_array_create(tiledb_ctx, &array_schema) == TILEDB_OK); CHECK(tiledb_array_free_schema(&array_schema) == TILEDB_OK); CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } void write_arrays(BenchmarkConfig* config, int i) { TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = config->get_temp_dir().c_str(); tiledb_config.write_method_ = config->io_write_mode_; REQUIRE(tiledb_ctx_init(&tiledb_ctx, &tiledb_config) == TILEDB_OK); TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, &tiledb_array, config->array_names_[i].c_str(), config->array_write_mode_, NULL, // entire domain NULL, // all attributes 0); // number of attributes REQUIRE(tiledb_array_write(tiledb_array, const_cast(config->buffers_.data()), config->buffer_sizes_.data()) == TILEDB_OK); CHECK(tiledb_array_finalize(tiledb_array) == TILEDB_OK); CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } void read_arrays(BenchmarkConfig* config, int i) { TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = config->get_temp_dir().c_str(); tiledb_config.read_method_ = config->io_read_mode_; REQUIRE(tiledb_ctx_init(&tiledb_ctx, &tiledb_config) == TILEDB_OK); TileDB_Array* tiledb_array; tiledb_array_init( tiledb_ctx, &tiledb_array, config->array_names_[i].c_str(), config->array_read_mode_, NULL, // entire domain NULL, // all attributes 0); // number of attributes // std::vector buffers; // std::vector buffer_sizes; // create_buffers(config, buffers, buffer_sizes, false); REQUIRE(tiledb_array_read(tiledb_array, config->buffers_.data(), config->buffer_sizes_.data()) == TILEDB_OK); CHECK(tiledb_array_finalize(tiledb_array) == TILEDB_OK); CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } // Benchmark Filesystem Stat Utils std::string get_io_write_mode(int mode) { switch(mode) { case 0: return "TILEDB_IO_WRITE"; default: std::cerr << "TILEDB_IO_MODE=" << std::to_string(mode) << "not recognized\n"; return ""; } } std::string get_io_read_mode(int mode) { switch(mode) { case 0: return "TILEDB_IO_MMAP"; case 1: return "TILEDB_IO_READ"; default: std::cerr << "TILEDB_IO_MODE=" << std::to_string(mode) << "not recognized\n"; return ""; } } std::string get_array_mode(int mode) { switch(mode) { case 0: return "TILEDB_ARRAY_READ"; case 1: return "TILEDB_ARRAY_READ_SORTED_COL"; case 2: return "TILEDB_ARRAY_READ_SORTED_ROW"; case 3: return "TILEDB_ARRAY_WRITE"; case 4: return "TILEDB_ARRAY_WRITE_SORTED_COL"; case 5: return "TILEDB_ARRAY_WRITE_SORTED_ROW"; case 6: return "TILEDB_ARRAY_WRITE_UNSORTED"; default: std::cerr << "TILEDB_ARRAY_MODE=" << std::to_string(mode) << "not recognized\n"; return ""; } } std::string get_name(const std::string& path) { auto found = path.rfind("/"); if (found == std::string::npos) { return path; } else { return path.substr(found+1); } } std::string readable_size(size_t size, bool human_readable_sizes) { if (human_readable_sizes) { std::vector suffix = { "B", "KB", "MB", "GB", "TB" }; auto i = 0u; while (size > 0) { if (i > suffix.size()) break; if (size/1024 == 0) { return std::to_string(size) + suffix[i]; } else { size /= 1024; i++; } } } return std::to_string(size); } std::string get_attribute_info(BenchmarkConfig* config, const std::string& filename) { std::string attribute_info; if (filename.find(TILEDB_COORDS) != std::string::npos) { attribute_info.append(" type=int64 dimensions=" + std::to_string(config->dimensions_.size())); return attribute_info; } for (auto i=0u; iattributes_.size(); i++) { if (filename.find(config->attribute_names_[i]) != std::string::npos) { if (config->attribute_data_types_[i] == TILEDB_INT32) { attribute_info.append(" type=int32"); } else if (config->attribute_data_types_[i] == TILEDB_INT64) { attribute_info.append(" type=int64"); } else if (config->attribute_data_types_[i] == TILEDB_CHAR) { attribute_info.append(" type=char"); } if (config->attribute_cell_val_nums_[i] == TILEDB_VAR_NUM) { if (filename.find("_var.") == std::string::npos) { attribute_info.append(" offset_file"); } else { attribute_info.append(" num_cells=var"); } } else { attribute_info.append(" num_cells="+std::to_string(config->attribute_cell_val_nums_[i])); break; } } } return attribute_info; } void print_array_schema(TileDB_CTX* tiledb_ctx, const std::string& array) { std::string array_schema_filename = array+'/'+TILEDB_ARRAY_SCHEMA_FILENAME; size_t buffer_size = file_size(tiledb_ctx, array_schema_filename); void *buffer = malloc(buffer_size); // Load array schema into buffer REQUIRE(read_file(tiledb_ctx, array_schema_filename, 0, buffer, buffer_size) == TILEDB_OK); // Initialize array schema from buffer ArraySchema* array_schema = new ArraySchema(NULL); REQUIRE(array_schema->deserialize(buffer, buffer_size) == TILEDB_OK); // Print the schema array_schema->print(); delete array_schema; free(buffer); std::cerr << "\n\n"; } void print_fragment_sizes(BenchmarkConfig* config, bool human_readable_sizes) { TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = config->get_temp_dir().c_str(); REQUIRE(tiledb_ctx_init(&tiledb_ctx, &tiledb_config) == TILEDB_OK); bool is_printed = false; for(auto array : get_dirs(tiledb_ctx, config->workspace_)) { if (!is_array(tiledb_ctx, array)) continue; std::cerr << "\n\nArray " << get_name(array) << ":\n"; if (config->print_array_schema_) { print_array_schema(tiledb_ctx, array); } for(auto fragment: get_dirs(tiledb_ctx, array)) { if (!is_fragment(tiledb_ctx, fragment)) continue; std::cerr << " Fragment " << get_name(fragment) << ":\n"; for (auto file: get_files(tiledb_ctx, fragment)) { std::string filename = get_name(file); if (filename.compare(TILEDB_FRAGMENT_FILENAME) == 0) continue; std::cerr << " File " << filename << get_attribute_info(config, filename) << " size=" << readable_size(file_size(tiledb_ctx, file), human_readable_sizes) << "\n"; is_printed = true; } if (is_printed) break; std::cerr << "\n"; } if (is_printed) break; } CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } #endif // TILEDB_TESTS_H genomicsdb-0.0~git20231212.9d7ddd0/test/include/c_api/000077500000000000000000000000001453617025200220225ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/include/c_api/c_api_array_schema_spec.h000066400000000000000000000056531453617025200270070ustar00rootroot00000000000000/** * @file c_api_array_schema_spec.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Declarations for testing the C API array schema spec. */ #ifndef __C_API_ARRAY_SCHEMA_SPEC_H__ #define __C_API_ARRAY_SCHEMA_SPEC_H__ #include "tiledb.h" #include "storage_posixfs.h" #include "catch.h" /** Test fixture for the array schema. */ class ArraySchemaTestFixture : TempDir { public: /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /** Workspace folder name. */ const std::string WORKSPACE = get_temp_dir() + "/schema_workspace/"; /** * Array name. * Format: (x_x). */ const std::string ARRAYNAME = "dense_test_100x100_10x10"; /* ********************************* */ /* CONSTRUCTOR/DESTRUCTOR */ /* ********************************* */ ArraySchemaTestFixture(); ~ArraySchemaTestFixture(); /* ********************************* */ /* PUBLIC METHODS */ /* ********************************* */ /** * Creates a dense array. * * @return TILEDB_OK on success and TILEDB_ERR on error. */ int create_dense_array(std::string array_name, int attribute_datatype, int compression_type); void check_dense_array(std::string array_name); /* ********************************* */ /* PUBLIC ATTRIBUTES */ /* ********************************* */ /** Array name. */ std::string array_name_; /** Array schema object under test. */ TileDB_ArraySchema array_schema_; /** True if the array schema is set. */ bool array_schema_set_; /** TileDB context. */ TileDB_CTX* tiledb_ctx_; PosixFS fs_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/test/include/c_api/c_api_dense_array_spec.h000066400000000000000000000216261453617025200266430ustar00rootroot00000000000000/** * @file c_api_dense_array_spec.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Declarations for testing the C API dense array spec. */ #ifndef __C_API_DENSE_ARRAY_SPEC_H__ #define __C_API_DENSE_ARRAY_SPEC_H__ #include "catch.h" #include "tiledb.h" /** Test fixture for dense array operations. */ class DenseArrayTestFixture : TempDir { public: /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /** Workspace folder name. */ const std::string WORKSPACE = get_temp_dir() + "/dense_workspace/"; /* ********************************* */ /* CONSTRUCTORS/DESTUCTORS */ /* ********************************* */ DenseArrayTestFixture(); ~DenseArrayTestFixture(); /* ********************************* */ /* PUBLIC METHODS */ /* ********************************* */ /** * Checks two buffers, one before and one after the updates. The updates * are given as function inputs and facilitate the check. * * @param buffer_before The buffer before the updates. * @param buffer_after The buffer after the updates. * @param buffer_updates_a1 The updated attribute values. * @param buffer_updates_coords The coordinates where the updates occurred. * @param domain_size_0 The domain size of the first dimension. * @param domain_size_1 The domain size of the second dimension. * @param update_num The number of updates. * @return True on success and false on error. */ static bool check_buffer_after_updates( const int* buffer_before, const int* buffer_after, const int* buffer_updates_a1, const int64_t* buffer_updates_coords, const int64_t domain_size_0, const int64_t domain_size_1, const int64_t update_num); /** * Creates a 1D dense array. * * @param attribute_type * @param tile_extent * @param domain_lo * @param domain_hi * @param cell_order The cell order. * @param tile_order The tile order. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int create_dense_array_1D( const int attribute_type, const int32_t tile_extent, const int32_t domain_lo, const int32_t domain_hi, const int cell_order, const int tile_order); /** * Creates a 2D dense array. * * @param tile_extent_0 The tile extent along the first dimension. * @param tile_extent_1 The tile extent along the second dimension. * @param domain_0_lo The smallest value of the first dimension domain. * @param domain_0_hi The largest value of the first dimension domain. * @param domain_1_lo The smallest value of the second dimension domain. * @param domain_1_hi The largest value of the second dimension domain. * @param capacity The tile capacity. * @param enable_compression If true, then GZIP compression is used. * @param cell_order The cell order. * @param tile_order The tile order. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int create_dense_array_2D( const int64_t tile_extent_0, const int64_t tile_extent_1, const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int64_t capacity, const bool enable_compression, const int cell_order, const int tile_order); /** * Generates a 1D buffer containing the cell values of a 2D array. * Each cell value equals (row index * total number of columns + col index). * * @param domain_size_0 The domain size of the first dimension. * @param domain_size_1 The domain size of the second dimension. * @return The created buffer of size domain_size_0*domain_size_1 integers. * Note that the function creates the buffer with 'new'. Make sure * to delete the returned buffer in the caller function. */ int* generate_1D_int_buffer( const int64_t domain_size_0, const int64_t domain_size_1); /** * Generates a 2D buffer containing the cell values of a 2D array. * Each cell value equals (row index * total number of columns + col index). * @param domain_size_0 The domain size of the first dimension. * @param domain_size_1 The domain size of the second dimension. * @return The created 2D buffer. Note that the function creates the buffer * with 'new'. Make sure to delete the returned buffer in the caller * function. */ int** generate_2D_buffer( const int64_t domain_size_0, const int64_t domain_size_1); /** * Reads a subarray oriented by the input boundaries and outputs the buffer * containing the attribute values of the corresponding cells. * * @param domain_0_lo The smallest value of the first dimension domain to * be read. * @param domain_0_hi The largest value of the first dimension domain to * be read. * @param domain_1_lo The smallest value of the second dimension domain to * be read. * @param domain_1_hi The largest value of the second dimension domain to * be read. * @param read_mode The read mode. * @return The buffer with the read values. Note that this function is * creating the buffer with 'new'. Therefore, make sure to properly delete * it in the caller. On error, it returns NULL. */ int* read_dense_array_2D( const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int read_mode); /** Sets the array name for the current test. */ void set_array_name(const char *); /** * Updates random locations in a dense array with the input domain sizes. * * @param domain_size_0 The domain size of the first dimension. * @param domain_size_1 The domain size of the second dimension. * @param udpate_num The number of updates to be performed. * @param seed The seed for the random generator. * @param buffers The buffers to be dispatched to the write command. * @param buffer_sizes The buffer sizes to be dispatched to the write command. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int update_dense_array_2D( const int64_t domain_size_0, const int64_t domain_size_1, int64_t update_num, int seed, void** buffers, const size_t* buffer_sizes); /** * Write to a 2D dense array tile by tile. The buffer is initialized * with row_id*domain_size_1+col_id values. * * @param domain_size_0 The domain size of the first dimension. * @param domain_size_1 The domain size of the second dimension. * @param tile_extent_0 The tile extent along the first dimension. * @param tile_extent_1 The tile extent along the second dimension. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int write_dense_array_by_tiles( const int64_t domain_size_0, const int64_t domain_size_1, const int64_t tile_extent_0, const int64_t tile_extent_1); /** * Writes a 2D dense subarray. * * @param subarray The subarray to focus on, given as a vector of low, high * values. * @param write_mode The write mode. * @param buffer The attribute buffer to be populated and written. * @param buffer_sizes The buffer sizes to be dispatched to the write command. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int write_dense_subarray_2D( int64_t* subarray, int write_mode, int* buffer, size_t* buffer_sizes); /* ********************************* */ /* PUBLIC ATTRIBUTES */ /* ********************************* */ /** Array name. */ std::string array_name_; /** Array schema object under test. */ TileDB_ArraySchema array_schema_; /** TileDB context. */ TileDB_CTX* tiledb_ctx_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/test/include/c_api/c_api_sparse_array_spec.h000066400000000000000000000116231453617025200270360ustar00rootroot00000000000000/** * @file c_api_sparse_array_spec.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Declarations for testing the C API sparse array spec. */ #ifndef __C_API_SPARSE_ARRAY_SPEC_H__ #define __C_API_SPARSE_ARRAY_SPEC_H__ #include "catch.h" #include "tiledb.h" class SparseArrayTestFixture : TempDir { public: /* ********************************* */ /* CONSTANTS */ /* ********************************* */ /** Workspace folder name. */ const std::string WORKSPACE = get_temp_dir() + "/sparse_workspace/"; /* ********************************* */ /* CONSTRUCTOR/DESTRUCTOR */ /* ********************************* */ SparseArrayTestFixture(); ~SparseArrayTestFixture(); /* ********************************* */ /* PUBLIC METHODS */ /* ********************************* */ /** * Creates a 2D sparse array. * * @param tile_extent_0 The tile extent along the first dimension. * @param tile_extent_1 The tile extent along the second dimension. * @param domain_0_lo The smallest value of the first dimension domain. * @param domain_0_hi The largest value of the first dimension domain. * @param domain_1_lo The smallest value of the second dimension domain. * @param domain_1_hi The largest value of the second dimension domain. * @param capacity The tile capacity. * @param enable_compression If true, then GZIP compression is used. * @param cell_order The cell order. * @param tile_order The tile order. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int create_sparse_array_2D( const int64_t tile_extent_0, const int64_t tile_extent_1, const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int64_t capacity, const bool enable_compression, const int cell_order, const int tile_order); /** * Reads a subarray oriented by the input boundaries and outputs the buffer * containing the attribute values of the corresponding cells. * * @param domain_0_lo The smallest value of the first dimension domain to * be read. * @param domain_0_hi The largest value of the first dimension domain to * be read. * @param domain_1_lo The smallest value of the second dimension domain to * be read. * @param domain_1_hi The largest value of the second dimension domain to * be read. * @param read_mode The read mode. * @return The buffer with the read values. Note that this function is * creating the buffer with 'new'. Therefore, make sure to properly delete * it in the caller. On error, it returns NULL. */ int* read_sparse_array_2D( const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int read_mode, const ssize_t expected_num_cells = -1); /** Sets the array name for the current test. */ void set_array_name(const char *); /** * Write random values in unsorted mode. The buffer is initialized with each * cell being equalt to row_id*domain_size_1+col_id. * * @param domain_size_0 The domain size of the first dimension. * @param domain_size_1 The domain size of the second dimension. * @return TILEDB_OK on success and TILEDB_ERR on error. */ int write_sparse_array_unsorted_2D( const int64_t domain_size_0, const int64_t domain_size_1); /* ********************************* */ /* PUBLIC ATTRIBUTES */ /* ********************************* */ /** Array name. */ std::string array_name_; /** Array schema object under test. */ TileDB_ArraySchema array_schema_; /** TileDB context. */ TileDB_CTX* tiledb_ctx_; }; #endif genomicsdb-0.0~git20231212.9d7ddd0/test/include/catch2/000077500000000000000000000000001453617025200221135ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/include/catch2/catch.h000066400000000000000000000121001453617025200233400ustar00rootroot00000000000000/** * @file catch.h * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Catch Unit Testing Header */ #ifndef __TILEDB_CATCH_H__ #define __TILEDB_CATCH_H__ #if CATCH2_MAJOR_VERSION == 3 #include using Catch::Matchers::Equals; using Catch::Matchers::Matches; using Catch::startsWith; using Catch::endsWith; using Catch::contains; using namespace Catch::Clara; #elif CATCH2_MAJOR_VERSION == 2 #define CATCH_CONFIG_RUNNER #include using Catch::Equals; using Catch::StartsWith; using Catch::EndsWith; using Catch::Contains; using Catch::Matches; using namespace Catch::clara; #else # error Could not figure out CATCH2_MAJOR_VERSION #endif #include "tiledb_utils.h" #include "regex" #define CHECK_RC(rc, expected) CHECK(rc == expected) std::string g_test_dir = ""; std::string g_benchmark_config = std::string(TILEDB_TEST_DIR)+"/inputs/benchmark.config"; class TempDir { public: TempDir() { create_temp_directory(); } ~TempDir() { TileDBUtils::delete_dir(get_temp_dir()); if (!delete_test_dir_in_destructor_.empty()) { TileDBUtils::delete_dir(delete_test_dir_in_destructor_); } } const std::string& get_temp_dir() { return tmp_dirname_; } private: std::string tmp_dirname_; std::string delete_test_dir_in_destructor_; std::string get_pathname(std::string path) { const size_t last_slash_idx = path.rfind('/'); if (last_slash_idx != std::string::npos) { return path.substr(last_slash_idx+1); } else { return path; } } std::string append_slash(std::string path) { if(path[path.size()]!='/') { return path+"/"; } else { return path; } } void create_temp_directory() { std::string dirname_pattern("TileDBTestXXXXXX"); if (g_test_dir.empty()) { // Posix Case. Use mkdtemp() here const char *tmp_dir = getenv("TMPDIR"); if (tmp_dir == NULL) { tmp_dir = P_tmpdir; // defined in stdio } assert(tmp_dir != NULL); tmp_dirname_ = mkdtemp(const_cast((append_slash(tmp_dir)+dirname_pattern).c_str())); } else { std::string temp; for (auto i=0; i<6; i++) temp = temp+char(rand()%26+'a'); tmp_dirname_ = TileDBUtils::append_path( g_test_dir, std::regex_replace(dirname_pattern, std::regex("XXXXXX"), temp)); if (!TileDBUtils::is_dir(g_test_dir)) { REQUIRE(TileDBUtils::create_dir(g_test_dir) == 0); delete_test_dir_in_destructor_ = g_test_dir; } if (!TileDBUtils::is_dir(tmp_dirname_)) { CHECK(TileDBUtils::create_dir(tmp_dirname_) == TILEDB_OK); } } } }; const std::string get_test_dir() { return g_test_dir; } std::string append(const std::string &temp_dir, const std::string &append) { std::size_t find = temp_dir.find('?'); if (find == std::string::npos) return temp_dir + append; else { return temp_dir.substr(0, find) + append + temp_dir.substr(find); } } int main( int argc, char* argv[] ) { Catch::Session session; // There must be exactly one instance // Build a new parser on top of Catch's auto cli = session.cli() // Get Catch's composite command line parser | Opt( g_test_dir, "test-dir-url" ) // bind variable to a new option, with a hint string ["--test-dir"] // the option names it will respond to ("Specify test dir, default is $TMPDIR if not specified") // description string for the help output | Opt ( g_benchmark_config, "/path/to/TileDB/test/inputs/benchmark.config") ["--benchmark-config"] ("Specify config for benchmmarking, default is /path/to/TileDB/test/inputs/benchmark.config"); // Now pass the new composite back to Catch so it uses that session.cli(cli); // Let Catch (using Clara) parse the command line int rc = session.applyCommandLine( argc, argv ); if(rc != 0) // Indicates a command line error return rc; return session.run(); } #endif // __TILEDB_CATCH_H__ genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/000077500000000000000000000000001453617025200206465ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/benchmark.config000066400000000000000000000046651453617025200240020ustar00rootroot00000000000000Array_Names=test_array1,test_array2 #Optional - Default is 0 #define TILEDB_IO_MMAP 0 #define TILEDB_IO_READ 1 #define TILEDB_IO_MPI 2 #define TILEDB_IO_WRITE 0 IO_Write_Mode=0 IO_Read_Mode=1 #Optional - Default is TILEDB_ARRAY_READ and TILEDB_ARRAY_WRITE #define TILEDB_ARRAY_READ 0 #define TILEDB_ARRAY_READ_SORTED_COL 1 #define TILEDB_ARRAY_READ_SORTED_ROW 2 #define TILEDB_ARRAY_WRITE 3 #define TILEDB_ARRAY_WRITE_SORTED_COL 4 #define TILEDB_ARRAY_WRITE_SORTED_ROW 5 #define TILEDB_ARRAY_WRITE_UNSORTED 6 Array_Write_Mode=3 Array_Read_Mode=0 #Optional - Default is 1 Fragments_Per_Array=1 Attribute_Names=test_attr1,test_attr2,test_attr3,test_attr4 # One type per attribute Attribute_Data_Types=int32,char,int32,int64 # -1 denotes TILEDB_VAR_NUM Attribute_Cell_Val_Num=1,-1,1,1 Dimension_Names=X,Y # lo,hi per dimension - int64_t is the hardcoded type Domain=0,21474836470000,0,21474836470000 #define TILEDB_NO_COMPRESSION 0 #define TILEDB_GZIP 1 #define TILEDB_ZSTD 2 #define TILEDB_LZ4 3 #define TILEDB_BLOSC 4 #define TILEDB_BLOSC_LZ4 5 #define TILEDB_BLOSC_LZ4HC 6 #define TILEDB_BLOSC_SNAPPY 7 #define TILEDB_BLOSC_ZLIB 8 #define TILEDB_BLOSC_ZSTD 9 #define TILEDB_RLE 10 #define TILEDB_DELTA_ENCODE 16 #define TILEDB_BIT_SHUFFLE 32 #define TILEDB_COMPRESSION_LEVEL_GZIP -1 #define TILEDB_COMPRESSION_LEVEL_ZSTD 1 #define TILEDB_COMPRESSION_LEVEL_BLOSC 5 #define TILEDB_COMPRESSION_LEVEL_LZ4 -1 Compression=1,1,0,35,1 Compression_Levels=-1,-1,0,0,-1 Offsets_Compression=0,17,0,0 Offsets_Compression_Levels=0,-1,0,0 #Optional - Default is 0 - allow TileDB choose an appropriate capacity Cells_Per_Tile=1000 #Optional - Default is 1024 #Cells_To_Write=10000000 #Optional - Default is 1024 #Cells_To_Read=10000000 # Optional - Default is 1(True) Print_Human_Readable_Sizes=1 Print_Array_Schema=1 genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_gdb_pre100_ws/000077500000000000000000000000001453617025200261335ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_gdb_pre100_ws/__tiledb_workspace.tdb000077500000000000000000000000001453617025200324360ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_gdb_pre100_ws/t0_1_2/000077500000000000000000000000001453617025200271175ustar00rootroot00000000000000.__consolidation_lock000077500000000000000000000000001453617025200332050ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_gdb_pre100_ws/t0_1_2genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_gdb_pre100_ws/t0_1_2/__array_schema.tdb000077500000000000000000000007051453617025200325530ustar00rootroot00000000000000/tmp/tmp7l5mFz/ws/t0_1_2ENDREFALTQUALFILTER BaseQRankSumClippingRankSum MQRankSumReadPosRankSumMQRAW_MQMQ0DP DP_FORMATGQSBADPLPGTPIDMIN_DPGTsamplesposition genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/000077500000000000000000000000001453617025200251605ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/000077500000000000000000000000001453617025200276635ustar00rootroot00000000000000__tiledb_workspace.tdb000077500000000000000000000000001453617025200341070ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspacegenomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/000077500000000000000000000000001453617025200323425ustar00rootroot00000000000000__tiledb_group.tdb000077500000000000000000000000001453617025200357240ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arraysmy_array_A/000077500000000000000000000000001453617025200343465ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays.__consolidation_lock000077500000000000000000000000001453617025200405130ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_A__4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/000077500000000000000000000000001453617025200433275ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_A__4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/__book_keeping.tdb.gz000077500000000000000000000000771453617025200474020ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_AS`F(͂ŅPDCJ+@iYS0__4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/__tiledb_fragment.tdb000077500000000000000000000000001453617025200474370ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_A__4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/a1.tdb000077500000000000000000000001141453617025200443220ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_Axc````b& fb8xca```b6 fbx```b. bx'xa``b> b7__4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/a2.tdb000077500000000000000000000001141453617025200443230ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_Axc`F( ٠4 xbn( 43xaQ(-4[xcy(U4 __4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/a2_var.tdb000077500000000000000000000001001453617025200451660ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_AxKLJJNNN4xKMKKOOx/x/W__4006A3C2-AA4A-47D5-A836-18A2F8C0F17E4555318720_1586969354458/a3.tdb000077500000000000000000000002141453617025200443250ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_Ax;{3>vgϚ9>-47 qx36nvHKks06^ @~ 9 mx5ؘqLA - 26Ǝ =x5q@  t2.Ŏ =__array_schema.tdb000077500000000000000000000002611453617025200377770ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/dense_arrays/my_array_A$my_workspace/dense_arrays/my_array_Aa1a2a3d1d2 genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/000077500000000000000000000000001453617025200325415ustar00rootroot00000000000000__tiledb_group.tdb000077500000000000000000000000001453617025200361230ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arraysmy_array_B/000077500000000000000000000000001453617025200345465ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays.__consolidation_lock000077500000000000000000000000001453617025200407130ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/000077500000000000000000000000001453617025200435755ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/__book_keeping.tdb.gz000077500000000000000000000001311453617025200476370ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_BS`F(͂.N33GRN4qtsU0eptPZJ+s7;> H I__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/__coords.tdb000077500000000000000000000002001453617025200460520ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/__tiledb_fragment.tdb000077500000000000000000000000001453617025200477050ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/a1.tdb000077500000000000000000000000701453617025200445710ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_Bxc````b xcb```b$xca```b< xcc```bT__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/a2.tdb000077500000000000000000000000701453617025200445720ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_Bxc`F( xcf6( p xbn( xe( `__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/a2_var.tdb000077500000000000000000000000601453617025200454410ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_BxKLJL&xKNNN xKMKd2xKOO U__6A0F5AA3-8FF8-4761-86AC-7F59F038C23E4594509248_1586969340972/a3.tdb000077500000000000000000000001001453617025200445650ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B=L>̌??ff@ @ffF@L@33@ff@33@ff@33@ff@33@ff@__array_schema.tdb000077500000000000000000000002621453617025200402000ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B%my_workspace/sparse_arrays/my_array_Ba1a2a3d1d2 genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/000077500000000000000000000000001453617025200231755ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/__tiledb_workspace.tdb000077500000000000000000000000001453617025200275000ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/000077500000000000000000000000001453617025200260535ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/__tiledb_group.tdb000077500000000000000000000000001453617025200315140ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/000077500000000000000000000000001453617025200301375ustar00rootroot00000000000000.__consolidation_lock000077500000000000000000000000001453617025200342250ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620/000077500000000000000000000000001453617025200370535ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B__book_keeping.tdb.gz000077500000000000000000000001421453617025200431170ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620S`F(͂.N33GRN4qtsU0e`] VFsCi (̀;> Iξ 0__coords.tdb000077500000000000000000000001031453617025200413320ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620xcdFT`xcd(C$xcfF4xcffT__tiledb_fragment.tdb000077500000000000000000000000001453617025200431630ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620a1.tdb000077500000000000000000000000701453617025200400470ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620xc````b xcb```b$xca```b< xcc```bTa2.tdb000077500000000000000000000000701453617025200400500ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620xc`F( xcff( XxbF( xef( a2_var.tdb000077500000000000000000000000601453617025200407170ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620xKLJL&xKNNN xKMKd2xKOO Ua3.tdb000077500000000000000000000001001453617025200400430ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B/__4FECD19A-4F7D-4D21-B170-A2D111C8FCC44550772224_1652300381620=L>̌??ff@ @ffF@L@33@ff@33@ff@33@ff@33@ff@__array_schema.tdb000077500000000000000000000002701453617025200335110ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/examples_ws/sparse_arrays/my_array_B%my_workspace/sparse_arrays/my_array_Ba1a2a3d1d2 !genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/000077500000000000000000000000001453617025200234715ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/000077500000000000000000000000001453617025200247415ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/.__consolidation_lock000077500000000000000000000000001453617025200311060ustar00rootroot00000000000000__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445/000077500000000000000000000000001453617025200337005ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621AD.tdb000077500000000000000000000000161453617025200346570ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` XAD_var.tdb000077500000000000000000000000341453617025200355270ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xb``bAbU bKALT.tdb000077500000000000000000000000211453617025200350070ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`F43  ALT_var.tdb000077500000000000000000000000221453617025200356600ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xSSSSsQ UBaseQRankSum.tdb000077500000000000000000000000331453617025200366660ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x?v kˁǭUClippingRankSum.tdb000077500000000000000000000000331453617025200374400ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x?j>w5'[DP.tdb000077500000000000000000000000251453617025200346760ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x4\K)1DP_FORMAT.tdb000077500000000000000000000000301453617025200357420ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xcb```b&  b DS.tdb000077500000000000000000000000151453617025200347000ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xcdzEND.tdb000077500000000000000000000000311453617025200347760ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc3`/J@iZFILTER.tdb000077500000000000000000000000161453617025200353600ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`  FILTER_var.tdb000077500000000000000000000000131453617025200362250ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`GQ.tdb000077500000000000000000000000161453617025200347020ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`@H d*GT.tdb000077500000000000000000000000161453617025200347050ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`iIGT_var.tdb000077500000000000000000000000231453617025200355530ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` 0B1 cHaplotypeScore.tdb000077500000000000000000000000171453617025200373350ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x? 6InbreedingCoeff.tdb000077500000000000000000000000171453617025200374050ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x? 6MIN_DP.tdb000077500000000000000000000000211453617025200353750ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`@aFN uMLEAC.tdb000077500000000000000000000000161453617025200352140ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` pMLEAC_var.tdb000077500000000000000000000000271453617025200360660ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xcd```b& fbFMLEAF.tdb000077500000000000000000000000161453617025200352170ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` pMLEAF_var.tdb000077500000000000000000000000341453617025200360670ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc``5s=C1fa?!MQ.tdb000077500000000000000000000000331453617025200347070ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x??1ê\''MQ0.tdb000077500000000000000000000000261453617025200347710ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x$![ MQRankSum.tdb000077500000000000000000000000341453617025200362110ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x?Pb_w#JkPGT.tdb000077500000000000000000000000161453617025200350250ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` 0PGT_var.tdb000077500000000000000000000000161453617025200356750ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x3141PID.tdb000077500000000000000000000000161453617025200350070ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` pPID_var.tdb000077500000000000000000000000241453617025200356560ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x3470ww42B)PL.tdb000077500000000000000000000000211453617025200347020ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` 8aPL_var.tdb000077500000000000000000000001031453617025200355530ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc` `j ˘p20p20Av20cf`*"P'\ QUAL.tdb000077500000000000000000000000331453617025200351340ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x?9(գ!@WRAW_MQ.tdb000077500000000000000000000000311453617025200354160ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x?f`b y=REF.tdb000077500000000000000000000000161453617025200350070ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xc`FiREF_var.tdb000077500000000000000000000000171453617025200356600ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445xsvvvvwwsReadPosRankSum.tdb000077500000000000000000000000331453617025200372300ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x?溾Z湽SB.tdb000077500000000000000000000000401453617025200346740ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445x 1`@,ļ@,g8]__book_keeping.tdb.gz000077500000000000000000000001531453617025200377460ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445S`@A=:Mч/!4:r &__tiledb_fragment.tdb000077500000000000000000000000001453617025200400100ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__3ACDA3EF-1068-447C-B57F-96E920E75E1A4518356480_1681829219445genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/__array_schema.tdb000077500000000000000000000012361453617025200303750ustar00rootroot00000000000000D/Users/nalini/TileDB.develop/test/inputs/genomicsdb_ws/1$1$249250621ENDREFALTQUALFILTERDP BaseQRankSumClippingRankSumDSHaplotypeScoreInbreedingCoeffMLEACMLEAFMQRAW_MQMQ0 MQRankSumReadPosRankSumPGTPIDAD DP_FORMATGQGTMIN_DPPLSBsamplesposition genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/genomicsdb_meta_dir/000077500000000000000000000000001453617025200307175ustar00rootroot00000000000000genomicsdb_column_bounds.json000077500000000000000000000000641453617025200365770ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/genomicsdb_meta_dir{ "min_column": 0, "max_column": 249250620 }genomicsdb_meta_703A9BE7-49B1-4BB0-9F5B-20793E75F2A6.json000077500000000000000000000000741453617025200407670ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/1$1$249250621/genomicsdb_meta_dir{ "lb_row_idx": 0, "max_valid_row_idx_in_array": 2 }genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/__tiledb_workspace.tdb000077500000000000000000000000001453617025200277740ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/callset.json000077500000000000000000000007131453617025200260170ustar00rootroot00000000000000{ "callsets": [ { "sample_name": "HG00141", "row_idx": 0, "idx_in_file": 0, "filename": "/Users/nalini/GenomicsDB/tests/inputs/vcfs/t0.vcf.gz" }, { "sample_name": "HG01958", "row_idx": 1, "idx_in_file": 0, "filename": "/Users/nalini/GenomicsDB/tests/inputs/vcfs/t1.vcf.gz" }, { "sample_name": "HG01530", "row_idx": 2, "idx_in_file": 0, "filename": "/Users/nalini/GenomicsDB/tests/inputs/vcfs/t2.vcf.gz" } ] } genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/loader.json000077500000000000000000000371011453617025200256370ustar00rootroot00000000000000{ "column_partitions": [ { "begin": { "tiledb_column": 0 }, "workspace": "../genomicsdb_ws", "array_name": "1$1$249250621", "end": { "tiledb_column": 249250620 } }, { "begin": { "tiledb_column": 249250621 }, "workspace": "../genomicsdb_ws", "array_name": "2$1$243199373", "end": { "tiledb_column": 492449993 } }, { "begin": { "tiledb_column": 492449994 }, "workspace": "../genomicsdb_ws", "array_name": "3$1$198022430", "end": { "tiledb_column": 690472423 } }, { "begin": { "tiledb_column": 690472424 }, "workspace": "../genomicsdb_ws", "array_name": "4$1$191154276", "end": { "tiledb_column": 881626699 } }, { "begin": { "tiledb_column": 881626700 }, "workspace": "../genomicsdb_ws", "array_name": "5$1$180915260", "end": { "tiledb_column": 1062541959 } }, { "begin": { "tiledb_column": 1062541960 }, "workspace": "../genomicsdb_ws", "array_name": "6$1$171115067", "end": { "tiledb_column": 1233657026 } }, { "begin": { "tiledb_column": 1233657027 }, "workspace": "../genomicsdb_ws", "array_name": "7$1$159138663", "end": { "tiledb_column": 1392795689 } }, { "begin": { "tiledb_column": 1392795690 }, "workspace": "../genomicsdb_ws", "array_name": "8$1$146364022", "end": { "tiledb_column": 1539159711 } }, { "begin": { "tiledb_column": 1539159712 }, "workspace": "../genomicsdb_ws", "array_name": "9$1$141213431", "end": { "tiledb_column": 1680373142 } }, { "begin": { "tiledb_column": 1680373143 }, "workspace": "../genomicsdb_ws", "array_name": "10$1$135534747", "end": { "tiledb_column": 1815907889 } }, { "begin": { "tiledb_column": 1815907890 }, "workspace": "../genomicsdb_ws", "array_name": "11$1$135006516", "end": { "tiledb_column": 1950914405 } }, { "begin": { "tiledb_column": 1950914406 }, "workspace": "../genomicsdb_ws", "array_name": "12$1$133851895", "end": { "tiledb_column": 2084766300 } }, { "begin": { "tiledb_column": 2084766301 }, "workspace": "../genomicsdb_ws", "array_name": "13$1$115169878", "end": { "tiledb_column": 2199936178 } }, { "begin": { "tiledb_column": 2199936179 }, "workspace": "../genomicsdb_ws", "array_name": "14$1$107349540", "end": { "tiledb_column": 2307285718 } }, { "begin": { "tiledb_column": 2307285719 }, "workspace": "../genomicsdb_ws", "array_name": "15$1$102531392", "end": { "tiledb_column": 2409817110 } }, { "begin": { "tiledb_column": 2409817111 }, "workspace": "../genomicsdb_ws", "array_name": "16$1$90354753", "end": { "tiledb_column": 2500171863 } }, { "begin": { "tiledb_column": 2500171864 }, "workspace": "../genomicsdb_ws", "array_name": "17$1$81195210", "end": { "tiledb_column": 2581367073 } }, { "begin": { "tiledb_column": 2581367074 }, "workspace": "../genomicsdb_ws", "array_name": "18$1$78077248", "end": { "tiledb_column": 2659444321 } }, { "begin": { "tiledb_column": 2659444322 }, "workspace": "../genomicsdb_ws", "array_name": "19$1$59128983", "end": { "tiledb_column": 2718573304 } }, { "begin": { "tiledb_column": 2718573305 }, "workspace": "../genomicsdb_ws", "array_name": "20$1$63025520", "end": { "tiledb_column": 2781598824 } }, { "begin": { "tiledb_column": 2781598825 }, "workspace": "../genomicsdb_ws", "array_name": "21$1$48129895", "end": { "tiledb_column": 2829728719 } }, { "begin": { "tiledb_column": 2829728720 }, "workspace": "../genomicsdb_ws", "array_name": "22$1$51304566", "end": { "tiledb_column": 2881033285 } }, { "begin": { "tiledb_column": 2881033286 }, "workspace": "../genomicsdb_ws", "array_name": "X$1$155270560", "end": { "tiledb_column": 3036303845 } }, { "begin": { "tiledb_column": 3036303846 }, "workspace": "../genomicsdb_ws", "array_name": "Y$1$59373566", "end": { "tiledb_column": 3095677411 } }, { "begin": { "tiledb_column": 3095677412 }, "workspace": "../genomicsdb_ws", "array_name": "MT$1$16569", "end": { "tiledb_column": 3095693980 } }, { "begin": { "tiledb_column": 3095693981 }, "workspace": "../genomicsdb_ws", "array_name": "GL000207.1$1$4262", "end": { "tiledb_column": 3095698242 } }, { "begin": { "tiledb_column": 3095698243 }, "workspace": "../genomicsdb_ws", "array_name": "GL000226.1$1$15008", "end": { "tiledb_column": 3095713250 } }, { "begin": { "tiledb_column": 3095713251 }, "workspace": "../genomicsdb_ws", "array_name": "GL000229.1$1$19913", "end": { "tiledb_column": 3095733163 } }, { "begin": { "tiledb_column": 3095733164 }, "workspace": "../genomicsdb_ws", "array_name": "GL000231.1$1$27386", "end": { "tiledb_column": 3095760549 } }, { "begin": { "tiledb_column": 3095760550 }, "workspace": "../genomicsdb_ws", "array_name": "GL000210.1$1$27682", "end": { "tiledb_column": 3095788231 } }, { "begin": { "tiledb_column": 3095788232 }, "workspace": "../genomicsdb_ws", "array_name": "GL000239.1$1$33824", "end": { "tiledb_column": 3095822055 } }, { "begin": { "tiledb_column": 3095822056 }, "workspace": "../genomicsdb_ws", "array_name": "GL000235.1$1$34474", "end": { "tiledb_column": 3095856529 } }, { "begin": { "tiledb_column": 3095856530 }, "workspace": "../genomicsdb_ws", "array_name": "GL000201.1$1$36148", "end": { "tiledb_column": 3095892677 } }, { "begin": { "tiledb_column": 3095892678 }, "workspace": "../genomicsdb_ws", "array_name": "GL000247.1$1$36422", "end": { "tiledb_column": 3095929099 } }, { "begin": { "tiledb_column": 3095929100 }, "workspace": "../genomicsdb_ws", "array_name": "GL000245.1$1$36651", "end": { "tiledb_column": 3095965750 } }, { "begin": { "tiledb_column": 3095965751 }, "workspace": "../genomicsdb_ws", "array_name": "GL000197.1$1$37175", "end": { "tiledb_column": 3096002925 } }, { "begin": { "tiledb_column": 3096002926 }, "workspace": "../genomicsdb_ws", "array_name": "GL000203.1$1$37498", "end": { "tiledb_column": 3096040423 } }, { "begin": { "tiledb_column": 3096040424 }, "workspace": "../genomicsdb_ws", "array_name": "GL000246.1$1$38154", "end": { "tiledb_column": 3096078577 } }, { "begin": { "tiledb_column": 3096078578 }, "workspace": "../genomicsdb_ws", "array_name": "GL000249.1$1$38502", "end": { "tiledb_column": 3096117079 } }, { "begin": { "tiledb_column": 3096117080 }, "workspace": "../genomicsdb_ws", "array_name": "GL000196.1$1$38914", "end": { "tiledb_column": 3096155993 } }, { "begin": { "tiledb_column": 3096155994 }, "workspace": "../genomicsdb_ws", "array_name": "GL000248.1$1$39786", "end": { "tiledb_column": 3096195779 } }, { "begin": { "tiledb_column": 3096195780 }, "workspace": "../genomicsdb_ws", "array_name": "GL000244.1$1$39929", "end": { "tiledb_column": 3096235708 } }, { "begin": { "tiledb_column": 3096235709 }, "workspace": "../genomicsdb_ws", "array_name": "GL000238.1$1$39939", "end": { "tiledb_column": 3096275647 } }, { "begin": { "tiledb_column": 3096275648 }, "workspace": "../genomicsdb_ws", "array_name": "GL000202.1$1$40103", "end": { "tiledb_column": 3096315750 } }, { "begin": { "tiledb_column": 3096315751 }, "workspace": "../genomicsdb_ws", "array_name": "GL000234.1$1$40531", "end": { "tiledb_column": 3096356281 } }, { "begin": { "tiledb_column": 3096356282 }, "workspace": "../genomicsdb_ws", "array_name": "GL000232.1$1$40652", "end": { "tiledb_column": 3096396933 } }, { "begin": { "tiledb_column": 3096396934 }, "workspace": "../genomicsdb_ws", "array_name": "GL000206.1$1$41001", "end": { "tiledb_column": 3096437934 } }, { "begin": { "tiledb_column": 3096437935 }, "workspace": "../genomicsdb_ws", "array_name": "GL000240.1$1$41933", "end": { "tiledb_column": 3096479867 } }, { "begin": { "tiledb_column": 3096479868 }, "workspace": "../genomicsdb_ws", "array_name": "GL000236.1$1$41934", "end": { "tiledb_column": 3096521801 } }, { "begin": { "tiledb_column": 3096521802 }, "workspace": "../genomicsdb_ws", "array_name": "GL000241.1$1$42152", "end": { "tiledb_column": 3096563953 } }, { "begin": { "tiledb_column": 3096563954 }, "workspace": "../genomicsdb_ws", "array_name": "GL000243.1$1$43341", "end": { "tiledb_column": 3096607294 } }, { "begin": { "tiledb_column": 3096607295 }, "workspace": "../genomicsdb_ws", "array_name": "GL000242.1$1$43523", "end": { "tiledb_column": 3096650817 } }, { "begin": { "tiledb_column": 3096650818 }, "workspace": "../genomicsdb_ws", "array_name": "GL000230.1$1$43691", "end": { "tiledb_column": 3096694508 } }, { "begin": { "tiledb_column": 3096694509 }, "workspace": "../genomicsdb_ws", "array_name": "GL000237.1$1$45867", "end": { "tiledb_column": 3096740375 } }, { "begin": { "tiledb_column": 3096740376 }, "workspace": "../genomicsdb_ws", "array_name": "GL000233.1$1$45941", "end": { "tiledb_column": 3096786316 } }, { "begin": { "tiledb_column": 3096786317 }, "workspace": "../genomicsdb_ws", "array_name": "GL000204.1$1$81310", "end": { "tiledb_column": 3096867626 } }, { "begin": { "tiledb_column": 3096867627 }, "workspace": "../genomicsdb_ws", "array_name": "GL000198.1$1$90085", "end": { "tiledb_column": 3096957711 } }, { "begin": { "tiledb_column": 3096957712 }, "workspace": "../genomicsdb_ws", "array_name": "GL000208.1$1$92689", "end": { "tiledb_column": 3097050400 } }, { "begin": { "tiledb_column": 3097050401 }, "workspace": "../genomicsdb_ws", "array_name": "GL000191.1$1$106433", "end": { "tiledb_column": 3097156833 } }, { "begin": { "tiledb_column": 3097156834 }, "workspace": "../genomicsdb_ws", "array_name": "GL000227.1$1$128374", "end": { "tiledb_column": 3097285207 } }, { "begin": { "tiledb_column": 3097285208 }, "workspace": "../genomicsdb_ws", "array_name": "GL000228.1$1$129120", "end": { "tiledb_column": 3097414327 } }, { "begin": { "tiledb_column": 3097414328 }, "workspace": "../genomicsdb_ws", "array_name": "GL000214.1$1$137718", "end": { "tiledb_column": 3097552045 } }, { "begin": { "tiledb_column": 3097552046 }, "workspace": "../genomicsdb_ws", "array_name": "GL000221.1$1$155397", "end": { "tiledb_column": 3097707442 } }, { "begin": { "tiledb_column": 3097707443 }, "workspace": "../genomicsdb_ws", "array_name": "GL000209.1$1$159169", "end": { "tiledb_column": 3097866611 } }, { "begin": { "tiledb_column": 3097866612 }, "workspace": "../genomicsdb_ws", "array_name": "GL000218.1$1$161147", "end": { "tiledb_column": 3098027758 } }, { "begin": { "tiledb_column": 3098027759 }, "workspace": "../genomicsdb_ws", "array_name": "GL000220.1$1$161802", "end": { "tiledb_column": 3098189560 } }, { "begin": { "tiledb_column": 3098189561 }, "workspace": "../genomicsdb_ws", "array_name": "GL000213.1$1$164239", "end": { "tiledb_column": 3098353799 } }, { "begin": { "tiledb_column": 3098353800 }, "workspace": "../genomicsdb_ws", "array_name": "GL000211.1$1$166566", "end": { "tiledb_column": 3098520365 } }, { "begin": { "tiledb_column": 3098520366 }, "workspace": "../genomicsdb_ws", "array_name": "GL000199.1$1$169874", "end": { "tiledb_column": 3098690239 } }, { "begin": { "tiledb_column": 3098690240 }, "workspace": "../genomicsdb_ws", "array_name": "GL000217.1$1$172149", "end": { "tiledb_column": 3098862388 } }, { "begin": { "tiledb_column": 3098862389 }, "workspace": "../genomicsdb_ws", "array_name": "GL000216.1$1$172294", "end": { "tiledb_column": 3099034682 } }, { "begin": { "tiledb_column": 3099034683 }, "workspace": "../genomicsdb_ws", "array_name": "GL000215.1$1$172545", "end": { "tiledb_column": 3099207227 } }, { "begin": { "tiledb_column": 3099207228 }, "workspace": "../genomicsdb_ws", "array_name": "GL000205.1$1$174588", "end": { "tiledb_column": 3099381815 } }, { "begin": { "tiledb_column": 3099381816 }, "workspace": "../genomicsdb_ws", "array_name": "GL000219.1$1$179198", "end": { "tiledb_column": 3099561013 } }, { "begin": { "tiledb_column": 3099561014 }, "workspace": "../genomicsdb_ws", "array_name": "GL000224.1$1$179693", "end": { "tiledb_column": 3099740706 } }, { "begin": { "tiledb_column": 3099740707 }, "workspace": "../genomicsdb_ws", "array_name": "GL000223.1$1$180455", "end": { "tiledb_column": 3099921161 } }, { "begin": { "tiledb_column": 3099921162 }, "workspace": "../genomicsdb_ws", "array_name": "GL000195.1$1$182896", "end": { "tiledb_column": 3100104057 } }, { "begin": { "tiledb_column": 3100104058 }, "workspace": "../genomicsdb_ws", "array_name": "GL000212.1$1$186858", "end": { "tiledb_column": 3100290915 } }, { "begin": { "tiledb_column": 3100290916 }, "workspace": "../genomicsdb_ws", "array_name": "GL000222.1$1$186861", "end": { "tiledb_column": 3100477776 } }, { "begin": { "tiledb_column": 3100477777 }, "workspace": "../genomicsdb_ws", "array_name": "GL000200.1$1$187035", "end": { "tiledb_column": 3100664811 } }, { "begin": { "tiledb_column": 3100664812 }, "workspace": "../genomicsdb_ws", "array_name": "GL000193.1$1$189789", "end": { "tiledb_column": 3100854600 } }, { "begin": { "tiledb_column": 3100854601 }, "workspace": "../genomicsdb_ws", "array_name": "GL000194.1$1$191469", "end": { "tiledb_column": 3101046069 } }, { "begin": { "tiledb_column": 3101046070 }, "workspace": "../genomicsdb_ws", "array_name": "GL000225.1$1$211173", "end": { "tiledb_column": 3101257242 } }, { "begin": { "tiledb_column": 3101257243 }, "workspace": "../genomicsdb_ws", "array_name": "GL000192.1$1$547496", "end": { "tiledb_column": 3101804738 } }, { "begin": { "tiledb_column": 3101804739 }, "workspace": "../genomicsdb_ws", "array_name": "NC_007605$1$171823", "end": { "tiledb_column": 3101976561 } } ], "vid_mapping_file": "../genomicsdb_ws/vidmap.json", "callset_mapping_file": "../genomicsdb_ws/callset.json", "size_per_column_partition": 43581440, "segment_size": 40, "num_cells_per_tile": 1000, "consolidate_tiledb_array_after_load": false, "enable_shared_posixfs_optimizations": false } genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/query.json000066400000000000000000000005121453617025200255270ustar00rootroot00000000000000{ "query_contig_intervals": [ { "contig": "1", "begin": 1, "end": 249250621 } ], "attributes": [ "REF", "ALT", "GT" ], "bypass_intersecting_intervals_phase": true, "query_filter": "GT[0] == 1", "array_name": "1$1$249250621" } genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/vcfheader.vcf000066400000000000000000000250751453617025200261310ustar00rootroot00000000000000##fileformat=VCFv4.2 ##FILTER= ##ALT= ##FILTER= ##FORMAT= ##FORMAT= ##FORMAT= ##FORMAT= ##FORMAT= ##FORMAT= ##FORMAT= ##FORMAT= ##FORMAT= ##GATKCommandLine= ##GVCFBlock=minGQ=0(inclusive),maxGQ=5(exclusive) ##GVCFBlock=minGQ=20(inclusive),maxGQ=60(exclusive) ##GVCFBlock=minGQ=5(inclusive),maxGQ=20(exclusive) ##GVCFBlock=minGQ=60(inclusive),maxGQ=2147483647(exclusive) ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##INFO= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##contig= ##reference=file:///seq/references/Homo_sapiens_assembly19/v1/Homo_sapiens_assembly19.fasta ##bcftools_viewVersion=0.2.0-rc7-156-g4bb6fd7-dirty+htslib-0.2.0-rc7-146-g32864c0-dirty ##bcftools_viewCommand=view -O z -o t0.vcf.gz t0.vcf ##bcftools_viewVersion=0.2.0-rc7-637-g0b8ef45-dirty+htslib-0.2.0-rc7-568-gf0ad83a ##bcftools_viewVersion=0.2.0-rc7-658-g21994b9+htslib-1.3-48-g047b6e5 ##bcftools_viewVersion=1.3.1-1-g5098939+htslib-1.3-168-gc396093 #CHROM POS ID REF ALT QUAL FILTER INFO genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/genomicsdb_ws/vidmap.json000077500000000000000000000274111453617025200256540ustar00rootroot00000000000000{ "fields": [ { "name": "PASS", "type": [ "Integer" ], "vcf_field_class": [ "FILTER" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "LowQual", "type": [ "Integer" ], "vcf_field_class": [ "FILTER" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "PGT", "type": [ "String" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "var" } ] }, { "name": "PID", "type": [ "String" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "var" } ] }, { "name": "AD", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "var" } ] }, { "name": "DP", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT", "INFO" ], "length": [ { "variable_length_descriptor": "1" } ], "VCF_field_combine_operation": "sum" }, { "name": "GQ", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "GT", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "PP" } ] }, { "name": "MIN_DP", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "PL", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "G" } ] }, { "name": "SB", "type": [ "Integer" ], "vcf_field_class": [ "FORMAT" ], "length": [ { "variable_length_descriptor": "4" } ] }, { "name": "BaseQRankSum", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "ClippingRankSum", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "DS", "type": [ "Flag" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "END", "type": [ "Integer" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ], "VCF_field_combine_operation": "sum" }, { "name": "HaplotypeScore", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "InbreedingCoeff", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "MLEAC", "type": [ "Integer" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "A" } ] }, { "name": "MLEAF", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "A" } ] }, { "name": "MQ", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "RAW_MQ", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "MQ0", "type": [ "Integer" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "MQRankSum", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] }, { "name": "ReadPosRankSum", "type": [ "Float" ], "vcf_field_class": [ "INFO" ], "length": [ { "variable_length_descriptor": "1" } ] } ], "contigs": [ { "name": "1", "length": 249250621, "tiledb_column_offset": 0 }, { "name": "2", "length": 243199373, "tiledb_column_offset": 249250621 }, { "name": "3", "length": 198022430, "tiledb_column_offset": 492449994 }, { "name": "4", "length": 191154276, "tiledb_column_offset": 690472424 }, { "name": "5", "length": 180915260, "tiledb_column_offset": 881626700 }, { "name": "6", "length": 171115067, "tiledb_column_offset": 1062541960 }, { "name": "7", "length": 159138663, "tiledb_column_offset": 1233657027 }, { "name": "8", "length": 146364022, "tiledb_column_offset": 1392795690 }, { "name": "9", "length": 141213431, "tiledb_column_offset": 1539159712 }, { "name": "10", "length": 135534747, "tiledb_column_offset": 1680373143 }, { "name": "11", "length": 135006516, "tiledb_column_offset": 1815907890 }, { "name": "12", "length": 133851895, "tiledb_column_offset": 1950914406 }, { "name": "13", "length": 115169878, "tiledb_column_offset": 2084766301 }, { "name": "14", "length": 107349540, "tiledb_column_offset": 2199936179 }, { "name": "15", "length": 102531392, "tiledb_column_offset": 2307285719 }, { "name": "16", "length": 90354753, "tiledb_column_offset": 2409817111 }, { "name": "17", "length": 81195210, "tiledb_column_offset": 2500171864 }, { "name": "18", "length": 78077248, "tiledb_column_offset": 2581367074 }, { "name": "19", "length": 59128983, "tiledb_column_offset": 2659444322 }, { "name": "20", "length": 63025520, "tiledb_column_offset": 2718573305 }, { "name": "21", "length": 48129895, "tiledb_column_offset": 2781598825 }, { "name": "22", "length": 51304566, "tiledb_column_offset": 2829728720 }, { "name": "X", "length": 155270560, "tiledb_column_offset": 2881033286 }, { "name": "Y", "length": 59373566, "tiledb_column_offset": 3036303846 }, { "name": "MT", "length": 16569, "tiledb_column_offset": 3095677412 }, { "name": "GL000207.1", "length": 4262, "tiledb_column_offset": 3095693981 }, { "name": "GL000226.1", "length": 15008, "tiledb_column_offset": 3095698243 }, { "name": "GL000229.1", "length": 19913, "tiledb_column_offset": 3095713251 }, { "name": "GL000231.1", "length": 27386, "tiledb_column_offset": 3095733164 }, { "name": "GL000210.1", "length": 27682, "tiledb_column_offset": 3095760550 }, { "name": "GL000239.1", "length": 33824, "tiledb_column_offset": 3095788232 }, { "name": "GL000235.1", "length": 34474, "tiledb_column_offset": 3095822056 }, { "name": "GL000201.1", "length": 36148, "tiledb_column_offset": 3095856530 }, { "name": "GL000247.1", "length": 36422, "tiledb_column_offset": 3095892678 }, { "name": "GL000245.1", "length": 36651, "tiledb_column_offset": 3095929100 }, { "name": "GL000197.1", "length": 37175, "tiledb_column_offset": 3095965751 }, { "name": "GL000203.1", "length": 37498, "tiledb_column_offset": 3096002926 }, { "name": "GL000246.1", "length": 38154, "tiledb_column_offset": 3096040424 }, { "name": "GL000249.1", "length": 38502, "tiledb_column_offset": 3096078578 }, { "name": "GL000196.1", "length": 38914, "tiledb_column_offset": 3096117080 }, { "name": "GL000248.1", "length": 39786, "tiledb_column_offset": 3096155994 }, { "name": "GL000244.1", "length": 39929, "tiledb_column_offset": 3096195780 }, { "name": "GL000238.1", "length": 39939, "tiledb_column_offset": 3096235709 }, { "name": "GL000202.1", "length": 40103, "tiledb_column_offset": 3096275648 }, { "name": "GL000234.1", "length": 40531, "tiledb_column_offset": 3096315751 }, { "name": "GL000232.1", "length": 40652, "tiledb_column_offset": 3096356282 }, { "name": "GL000206.1", "length": 41001, "tiledb_column_offset": 3096396934 }, { "name": "GL000240.1", "length": 41933, "tiledb_column_offset": 3096437935 }, { "name": "GL000236.1", "length": 41934, "tiledb_column_offset": 3096479868 }, { "name": "GL000241.1", "length": 42152, "tiledb_column_offset": 3096521802 }, { "name": "GL000243.1", "length": 43341, "tiledb_column_offset": 3096563954 }, { "name": "GL000242.1", "length": 43523, "tiledb_column_offset": 3096607295 }, { "name": "GL000230.1", "length": 43691, "tiledb_column_offset": 3096650818 }, { "name": "GL000237.1", "length": 45867, "tiledb_column_offset": 3096694509 }, { "name": "GL000233.1", "length": 45941, "tiledb_column_offset": 3096740376 }, { "name": "GL000204.1", "length": 81310, "tiledb_column_offset": 3096786317 }, { "name": "GL000198.1", "length": 90085, "tiledb_column_offset": 3096867627 }, { "name": "GL000208.1", "length": 92689, "tiledb_column_offset": 3096957712 }, { "name": "GL000191.1", "length": 106433, "tiledb_column_offset": 3097050401 }, { "name": "GL000227.1", "length": 128374, "tiledb_column_offset": 3097156834 }, { "name": "GL000228.1", "length": 129120, "tiledb_column_offset": 3097285208 }, { "name": "GL000214.1", "length": 137718, "tiledb_column_offset": 3097414328 }, { "name": "GL000221.1", "length": 155397, "tiledb_column_offset": 3097552046 }, { "name": "GL000209.1", "length": 159169, "tiledb_column_offset": 3097707443 }, { "name": "GL000218.1", "length": 161147, "tiledb_column_offset": 3097866612 }, { "name": "GL000220.1", "length": 161802, "tiledb_column_offset": 3098027759 }, { "name": "GL000213.1", "length": 164239, "tiledb_column_offset": 3098189561 }, { "name": "GL000211.1", "length": 166566, "tiledb_column_offset": 3098353800 }, { "name": "GL000199.1", "length": 169874, "tiledb_column_offset": 3098520366 }, { "name": "GL000217.1", "length": 172149, "tiledb_column_offset": 3098690240 }, { "name": "GL000216.1", "length": 172294, "tiledb_column_offset": 3098862389 }, { "name": "GL000215.1", "length": 172545, "tiledb_column_offset": 3099034683 }, { "name": "GL000205.1", "length": 174588, "tiledb_column_offset": 3099207228 }, { "name": "GL000219.1", "length": 179198, "tiledb_column_offset": 3099381816 }, { "name": "GL000224.1", "length": 179693, "tiledb_column_offset": 3099561014 }, { "name": "GL000223.1", "length": 180455, "tiledb_column_offset": 3099740707 }, { "name": "GL000195.1", "length": 182896, "tiledb_column_offset": 3099921162 }, { "name": "GL000212.1", "length": 186858, "tiledb_column_offset": 3100104058 }, { "name": "GL000222.1", "length": 186861, "tiledb_column_offset": 3100290916 }, { "name": "GL000200.1", "length": 187035, "tiledb_column_offset": 3100477777 }, { "name": "GL000193.1", "length": 189789, "tiledb_column_offset": 3100664812 }, { "name": "GL000194.1", "length": 191469, "tiledb_column_offset": 3100854601 }, { "name": "GL000225.1", "length": 211173, "tiledb_column_offset": 3101046070 }, { "name": "GL000192.1", "length": 547496, "tiledb_column_offset": 3101257243 }, { "name": "NC_007605", "length": 171823, "tiledb_column_offset": 3101804739 } ] } genomicsdb-0.0~git20231212.9d7ddd0/test/inputs/valgrind.supp000066400000000000000000000152741453617025200233760ustar00rootroot00000000000000{ Memcheck:Leak match-leak-kinds: reachable fun:malloc obj:/usr/lib64/libgomp.so.1.0.0 obj:/usr/lib64/libgomp.so.1.0.0 obj:/usr/lib64/libgomp.so.1.0.0 fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN20GenericGrowableArray12raw_allocateEi fun:_GLOBAL__sub_I_jvmtiRawMonitor.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_c1_LinearScan.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_c1_LinearScan.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_c1_LinearScan.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_c1_LinearScan.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_c1_LinearScan.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_c1_LinearScan.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN20GenericGrowableArray12raw_allocateEi fun:_GLOBAL__sub_I_reflectionUtils.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_jvmtiRawMonitor.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_memoryService.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_memoryService.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN11ResourceObjnwEmNS_15allocation_typeE10MemoryType fun:_GLOBAL__sub_I_reflectionUtils.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN20GenericGrowableArray12raw_allocateEi fun:_GLOBAL__sub_I_memoryService.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN20GenericGrowableArray12raw_allocateEi fun:_GLOBAL__sub_I_memoryService.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEm fun:_GLOBAL__sub_I_decoder.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEm fun:_GLOBAL__sub_I_diagnosticFramework.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEm fun:_GLOBAL__sub_I_g1CollectedHeap.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEm fun:_GLOBAL__sub_I_jfrThreadSampler.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEm fun:_GLOBAL__sub_I_metaspace.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType7EEnwEm fun:_GLOBAL__sub_I_shenandoahParallelCleaning.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } { Memcheck:Leak match-leak-kinds: reachable fun:malloc fun:_ZN2os6mallocEm10MemoryTypeRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType4EEnwEmRK15NativeCallStack fun:_ZN8CHeapObjIL10MemoryType4EEnwEm fun:_GLOBAL__sub_I_codeCache.cpp fun:_dl_init obj:/usr/lib64/ld-2.17.so } genomicsdb-0.0~git20231212.9d7ddd0/test/src/000077500000000000000000000000001453617025200201135ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/src/array/000077500000000000000000000000001453617025200212315ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/src/array/test_array_schema.cc000066400000000000000000000204301453617025200252340ustar00rootroot00000000000000/** * @file test_array_schema.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for array schema. */ #include "array_schema.h" #include "utils.h" #include "storage_posixfs.h" #include #include "catch.h" class ArraySchemaTestClass { public: const char *ws = "this_workspace"; const char *array = "this_array"; const char* attributes[1] = {"this_attribute"}; const char* dimensions[1] = { "one_dim" }; int32_t domain[2] = {1, 100}; int32_t tile_extents[1] = {25}; int types[2] = {TILEDB_INT32, TILEDB_INT32}; ArraySchemaC c_schema; void init() { c_schema.array_workspace_= const_cast(ws); c_schema.array_name_= const_cast(array); c_schema.attributes_= const_cast(attributes); c_schema.attribute_num_ = 1; c_schema.capacity_= 0; c_schema.cell_order_= TILEDB_ROW_MAJOR; c_schema.cell_val_num_= NULL; c_schema.compression_= {0}; c_schema.compression_level_ = NULL; c_schema.offsets_compression_ = NULL; c_schema.offsets_compression_level_ = NULL; c_schema.dimensions_ = const_cast(dimensions); c_schema.dim_num_= 1; c_schema.domain_= domain; c_schema.tile_extents_= tile_extents; c_schema.tile_order_= TILEDB_ROW_MAJOR; c_schema.types_= types; } }; class ArraySchemaTestClass64BitDim { public: const char *ws = "this_workspace"; const char *array = "this_array_64"; const char* attributes[1] = {"this_attribute"}; const char* dimensions[1] = { "one_dim_64" }; int64_t domain[2] = {1, 1<33}; int64_t tile_extents[1] = {25}; int types[2] = {TILEDB_INT64, TILEDB_INT64}; ArraySchemaC c_schema; void init() { c_schema.array_workspace_= const_cast(ws); c_schema.array_name_= const_cast(array); c_schema.attributes_= const_cast(attributes); c_schema.attribute_num_ = 1; c_schema.capacity_= 0; c_schema.cell_order_= TILEDB_ROW_MAJOR; c_schema.cell_val_num_= NULL; c_schema.compression_= {0}; c_schema.compression_level_ = NULL; c_schema.offsets_compression_ = NULL; c_schema.offsets_compression_level_ = NULL; c_schema.dimensions_ = const_cast(dimensions); c_schema.dim_num_= 1; c_schema.domain_= domain; c_schema.tile_extents_= tile_extents; c_schema.tile_order_= TILEDB_ROW_MAJOR; c_schema.types_= types; } }; TEST_CASE("Test Array Schema Version", "[array_schema_version]") { ArraySchema schema(NULL); CHECK(schema.version_tag_exists()); CHECK(schema.get_version() == TILEDB_ARRAY_SCHEMA_VERSION_TAG); } TEST_CASE("Test Array Schema", "[array_schema]") { ArraySchema schema(new PosixFS()); schema.set_array_workspace(NULL); schema.set_array_name(NULL); CHECK(schema.set_attributes(NULL, 0) != TILEDB_OK); const char* attr1[1] = {"test_attr"}; CHECK(schema.set_attributes(const_cast(attr1), 0) != TILEDB_OK); CHECK(schema.set_attributes(const_cast(attr1), 1) == TILEDB_OK); const char* attr2[2] = {"test_attr", "test_attr"}; CHECK(schema.set_attributes(const_cast(attr2), 2) != TILEDB_OK); const char* attr3[2] = {"test_attr", "test_attr1"}; CHECK(schema.set_attributes(const_cast(attr3), 2) == TILEDB_OK); schema.set_capacity(100); schema.set_capacity(0); schema.set_cell_val_num(NULL); const int cell_val_num[2] = {1, TILEDB_VAR_NUM}; schema.set_cell_val_num(const_cast(cell_val_num)); CHECK(schema.set_cell_order(-100) != TILEDB_OK); CHECK(schema.set_cell_order(TILEDB_ROW_MAJOR) == TILEDB_OK); CHECK(schema.set_cell_order(TILEDB_COL_MAJOR) == TILEDB_OK); CHECK(schema.set_compression(NULL) == TILEDB_OK); CHECK(schema.set_compression_level(NULL) == TILEDB_OK); CHECK(schema.set_offsets_compression(NULL) == TILEDB_OK); CHECK(schema.set_offsets_compression_level(NULL) == TILEDB_OK); const int compression[3] = {TILEDB_GZIP, TILEDB_GZIP, TILEDB_GZIP}; CHECK(schema.set_compression(const_cast(compression)) == TILEDB_OK); CHECK(schema.set_offsets_compression(NULL) == TILEDB_OK); CHECK(schema.set_offsets_compression(const_cast(compression)) == TILEDB_OK); const int offsets_compression[2] = {0, 0}; CHECK(schema.set_offsets_compression(const_cast(offsets_compression)) != TILEDB_OK); const int offsets_compression1[2] = {0, TILEDB_GZIP+TILEDB_DELTA_ENCODE}; CHECK(schema.set_offsets_compression(const_cast(offsets_compression1)) == TILEDB_OK); CHECK(schema.set_dimensions(NULL, 0) != TILEDB_OK); const char* dim1[1] = {"dim1"}; CHECK(schema.set_dimensions(const_cast(dim1), 0) != TILEDB_OK); CHECK(schema.set_dimensions(const_cast(dim1), 1) == TILEDB_OK); const char* dim2[2] = {"dim1", "dim1"}; CHECK(schema.set_dimensions(const_cast(dim2), 2) != TILEDB_OK); CHECK(schema.set_dimensions(const_cast(attr1), 1) != TILEDB_OK); const char* dim3[2] = {"dim1", "dim2"}; CHECK(schema.set_dimensions(const_cast(dim3), 2) == TILEDB_OK); CHECK(schema.set_types(NULL) != TILEDB_OK); int attr_types1[2] = {-100, -100}; CHECK(schema.set_types(attr_types1) != TILEDB_OK); int attr_types2[3] = {TILEDB_INT32, TILEDB_INT32, TILEDB_INT32}; CHECK(schema.set_types(attr_types2) == TILEDB_OK); CHECK(schema.set_domain(NULL) != TILEDB_OK); int domain[4] = {0,99,0,99}; CHECK(schema.set_domain(domain) == TILEDB_OK); schema.set_dense(0); CHECK(schema.set_tile_extents(NULL) == TILEDB_OK); CHECK(schema.set_tile_order(-1) != TILEDB_OK); schema.print(); } TEST_CASE_METHOD(ArraySchemaTestClass, "Test Array Schema Print", "[array_schema_print]") { init(); ArraySchema schema(NULL); CHECK(schema.init(&c_schema) == TILEDB_OK); schema.print(); } TEST_CASE_METHOD(ArraySchemaTestClass, "Test Array Schema Init With Print", "[array_schema_init_with_print]") { init(); ArraySchema schema(NULL); CHECK(schema.init(&c_schema, true) == TILEDB_OK); } TEST_CASE_METHOD(ArraySchemaTestClass, "Test Array Schema Serialize", "[array_schema_serialize]") { init(); ArraySchema schema(NULL); CHECK(schema.init(&c_schema) == TILEDB_OK); void *array_schema_serialized; size_t array_schema_serialized_size; CHECK(schema.serialize(array_schema_serialized, array_schema_serialized_size) == TILEDB_OK); CHECK(array_schema_serialized_size > 0); ArraySchema schema_for_deserialize(NULL); CHECK(schema_for_deserialize.deserialize(array_schema_serialized, array_schema_serialized_size) == TILEDB_OK); } TEST_CASE_METHOD(ArraySchemaTestClass64BitDim, "Test Array Schema Print 64-Bit", "[array_schema_print_64]") { init(); ArraySchema schema(NULL); CHECK(schema.init(&c_schema) == TILEDB_OK); schema.print(); } TEST_CASE_METHOD(ArraySchemaTestClass64BitDim, "Test Array Schema Serialize 64-Bit", "[array_schema_serialize_64]") { init(); ArraySchema schema(NULL); CHECK(schema.init(&c_schema) == TILEDB_OK); void *array_schema_serialized; size_t array_schema_serialized_size; CHECK(schema.serialize(array_schema_serialized, array_schema_serialized_size) == TILEDB_OK); CHECK(array_schema_serialized_size > 0); ArraySchema schema_for_deserialize(NULL); CHECK(schema_for_deserialize.deserialize(array_schema_serialized, array_schema_serialized_size) == TILEDB_OK); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/benchmark/000077500000000000000000000000001453617025200220455ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/src/benchmark/test_print_array_schema.cc000066400000000000000000000061521453617025200272710ustar00rootroot00000000000000/** * @file test_array_print_schema.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020,2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Given a File or Cloud URL to a TileDB array, print out its array schema */ #include "array_schema.h" #include "tiledb.h" #include "tiledb_storage.h" #include #include #include #define CHECK_RC(...) \ do { \ int rc = __VA_ARGS__; \ if (rc) { \ printf("%s", &tiledb_errmsg[0]); \ printf("[PrintArraySchema::%s] Runtime Error.\n", __FILE__); \ return rc; \ } \ } while (false) int main(int argc, char *argv[]) { if (argc < 2) { std::cerr << "No arguments specified\n"; std::cerr << "Usage: test_print_array_schema /path/to/TileDBArray\n"; return 0; } std::string array(argv[1]); std::string array_schema_filename(array+'/'+TILEDB_ARRAY_SCHEMA_FILENAME); TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config = {}; tiledb_config.home_ = array.c_str(); CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config)); if (!is_array(tiledb_ctx, array)) { std::cerr << "Array " << array << " does not seem to be a TileDB Array. Exiting\n"; return 1; } size_t buffer_size = file_size(tiledb_ctx, array_schema_filename); void *buffer = malloc(buffer_size); // Load array schema into buffer CHECK_RC(read_file(tiledb_ctx, array_schema_filename, 0, buffer, buffer_size)); // Initialize array schema from buffer ArraySchema* array_schema = new ArraySchema(NULL); CHECK_RC(array_schema->deserialize(buffer, buffer_size)); // Print the schema array_schema->print(); delete array_schema; free(buffer); std::cerr << "\n\n"; // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx)); return 0; } genomicsdb-0.0~git20231212.9d7ddd0/test/src/benchmark/test_print_book_keeping.cc000066400000000000000000000222731453617025200272710ustar00rootroot00000000000000/** * @file test_array_print_book_keeping.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Given a File or Cloud URL to a TileDB fragment, print out the book-keeping contents */ #include "array_schema.h" #include "book_keeping.h" #include "storage_manager_config.h" #include "tiledb.h" #include "tiledb_storage.h" #include "tiledb_utils.h" #include #include #include #define CHECK_RC(...) \ do { \ int rc = __VA_ARGS__; \ if (rc) { \ printf("%s", &tiledb_errmsg[0]); \ printf("[PrintBookKeeping::%s] Runtime Error.\n", __FILE__); \ return rc; \ } \ } while (false) ArraySchema* array_schema; std::shared_ptr bk_buffer; int attribute_num = -1; int print_non_empty_domain() { // Get domain size size_t domain_size; CHECK_RC(bk_buffer->read_buffer(&domain_size, sizeof(size_t))); std::cerr << "Domain size=" << domain_size << std::endl; // Get non-empty domain, assuming domain to be int64_t for now int64_t domain[domain_size/sizeof(size_t)]; if(domain_size != 0) { CHECK_RC(bk_buffer->read_buffer(&domain[0], domain_size)); std::cerr << "Domain coords ="; for (auto i = 0ul; iexpand_domain(domain); int64_domain = static_cast(domain); std::cerr << "Expanded domain coords ="; for (auto i = 0; icoords_size(); // Get number of MBRs int64_t mbr_num; CHECK_RC(bk_buffer->read_buffer(&mbr_num, sizeof(int64_t))); std::cerr << "Number of mbrs=" << mbr_num << std::endl; // Get MBRs std::cerr << "MBRs :" << std::endl; int64_t mbr[mbr_size/sizeof(size_t)]; for(int64_t i=0; iread_buffer(&mbr[0], mbr_size)); for (auto j = 0ul; jcoords_size(); // Get number of bounding coordinates int64_t bounding_coords_num; CHECK_RC(bk_buffer->read_buffer(&bounding_coords_num, sizeof(int64_t))); std::cerr << "Number of bounding coords=" << bounding_coords_num << std::endl; // Get bounding coordinates std::cerr << "Bounding Coords :" << std::endl; int64_t bounding_coords[bounding_coords_size/sizeof(int64_t)]; for(int64_t i=0; iread_buffer(&bounding_coords[0], bounding_coords_size)); for (auto j = 0ul; jread_buffer(&tile_offsets_num, sizeof(int64_t))); std::cerr << "Tile offsets for attribute=" << array_schema->attribute(i) << " tile_offsets_num=" << tile_offsets_num << std::endl; if(tile_offsets_num == 0) { continue; } // Get tile offsets off_t tile_offsets[tile_offsets_num]; CHECK_RC(bk_buffer->read_buffer(&tile_offsets[0], tile_offsets_num * sizeof(off_t))); for (auto j = 0; jread_buffer(&tile_var_offsets_num, sizeof(int64_t))); std::cerr << "Tile var offsets for attribute=" << array_schema->attribute(i) << " tile_var_offsets_num=" << tile_var_offsets_num << std::endl; if(tile_var_offsets_num == 0) continue; // Get variable tile offsets off_t tile_var_offsets[tile_var_offsets_num]; CHECK_RC(bk_buffer->read_buffer(&tile_var_offsets[0], tile_var_offsets_num * sizeof(off_t))); for (auto j = 0; jread_buffer(&tile_var_sizes_num, sizeof(int64_t))); std::cerr << "Tile var sizes for attribute=" << array_schema->attribute(i) << " tile_var_sizes_num=" << tile_var_sizes_num << std::endl; if(tile_var_sizes_num == 0) continue; // Get variable tile sizes size_t tile_var_sizes[tile_var_sizes_num]; CHECK_RC(bk_buffer->read_buffer(&tile_var_sizes[0], tile_var_sizes_num * sizeof(size_t))); for (auto j = 0; jread_buffer(&last_tile_cell_num, sizeof(int64_t))); std::cerr << "Last Tile cell num=" << last_tile_cell_num << std::endl; // Success return TILEDB_OK; } int main(int argc, char *argv[]) { if (argc < 2) { std::cerr << "No arguments specified\n"; std::cerr << "Usage: test_print_book_keeping /path/to/TileDBArray/\n"; return 0; } std::string array_fragment(argv[1]); // from argv[1] if (!TileDBUtils::is_file(array_fragment+"/"+TILEDB_FRAGMENT_FILENAME)) { std::cerr << "Specified input=\"" << array_fragment << "\" does not seem to be a fragment" << std::endl; return -1; } if (array_fragment.find("://") == std::string::npos) { PosixFS fs; array_fragment = fs.real_dir(array_fragment); } // Get Array Schema std::string array_name = parent_dir(array_fragment); std::string array_schema_filename(array_name+'/'+TILEDB_ARRAY_SCHEMA_FILENAME); void *buffer; size_t buffer_size; CHECK_RC(TileDBUtils::read_entire_file(array_name+'/'+TILEDB_ARRAY_SCHEMA_FILENAME, &buffer, &buffer_size)); array_schema = new ArraySchema(NULL); CHECK_RC(array_schema->deserialize(buffer, buffer_size)); free(buffer); // Get Storage Filesystem StorageManagerConfig config; config.init(argv[1]); StorageFS* fs = config.get_filesystem(); // Load Bookkeeping std::string book_keeping_filename = array_fragment+'/'+TILEDB_BOOK_KEEPING_FILENAME+TILEDB_FILE_SUFFIX+TILEDB_GZIP_SUFFIX; std::shared_ptr book_keeping = std::make_shared(array_schema, array_schema->dense(), array_fragment, TILEDB_ARRAY_READ); bk_buffer = std::make_shared(fs, book_keeping_filename, 1024, true, TILEDB_GZIP, TILEDB_COMPRESSION_LEVEL_GZIP); // For easy reference attribute_num = array_schema->attribute_num(); int rc = print_non_empty_domain(); rc |= print_mbrs(); rc |= print_bounding_coords(); rc |= print_tile_offsets(); rc |= print_tile_var_offsets(); rc |= print_tile_var_sizes(); rc |= print_last_tile_cell_num(); if (rc) { std::cerr << "Something went wrong!!" << std::endl; } delete array_schema; return rc; } genomicsdb-0.0~git20231212.9d7ddd0/test/src/benchmark/test_sparse_array_benchmark.cc000066400000000000000000000075201453617025200301240ustar00rootroot00000000000000/** * @file test_sparse_array.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Benchmark Sparse Arrays */ #define CATCH_CONFIG_ENABLE_BENCHMARKING #include "catch.h" #include "tiledb_benchmark.h" #include #include #include #include TEST_CASE_METHOD(BenchmarkConfig, "Benchmark sparse array", "[benchmark_sparse]") { if (!do_benchmark_) { return; } create_workspace(this); // Create Arrays std::cout << "Number of arrays to create=" << std::to_string(array_names_.size()) << std::endl; Catch::Timer t; t.start(); std::vector threads; for (auto i=0ul; i 1) { std::cout << " mean time = " << total_elapsed_time/fragments_per_array_ << "ms" << std::endl; } // Read Arrays std::cout << "\nNumber of cells to write= " << std::to_string(num_cells_to_read_) << std::endl; threads.clear(); create_buffers(false); for (auto i=0ul; i= 0); CHECK_RC(tiledb_array_iterator_get_value(tiledb_array_iterator, 1, (const void **)&coords, &coords_size), TILEDB_OK); CHECK(coords_size == 16); bool cond = *coords >= 0 && *(coords+1) >= 0; CHECK(cond); CHECK(*coords == *(coords+1)); if (!contains_empty_attributes) { CHECK(*a1_value == *coords); } CHECK_RC(tiledb_array_iterator_next(tiledb_array_iterator), TILEDB_OK); i++; } CHECK(i == nval); } void finalize_array_iterator() { CHECK_RC(tiledb_array_iterator_finalize(tiledb_array_iterator), TILEDB_OK); } }; TEST_CASE_METHOD(ArrayIteratorFixture, "Test sparse array iterator", "[sparse_array_iterator_full]") { create_sparse_array("test_sparse_array_it_full"); setup_array_iterator(); check_array_iterator(8); finalize_array_iterator(); } TEST_CASE_METHOD(ArrayIteratorFixture, "Test sparse array iterator with reset", "[sparse_array_iterator_with_reset]") { create_sparse_array("test_sparse_array_it_reset"); setup_array_iterator(); check_array_iterator(8); int64_t subarray[] = { 0, 7, 0, 7 }; CHECK_RC(tiledb_array_iterator_reset_subarray(tiledb_array_iterator, subarray), TILEDB_OK); check_array_iterator(8); finalize_array_iterator(); } TEST_CASE_METHOD(ArrayIteratorFixture, "Test sparse array iterator with reset subarray", "[sparse_array_iterator_with_reset_subarray]") { create_sparse_array("test_sparse_array_it_reset_subarray"); setup_array_iterator(); check_array_iterator(8); int64_t subarray[] = { 0, 4, 0, 4 }; CHECK_RC(tiledb_array_iterator_reset_subarray(tiledb_array_iterator, subarray), TILEDB_OK); check_array_iterator(5); finalize_array_iterator(); } TEST_CASE_METHOD(ArrayIteratorFixture, "Test sparse array iterator with filter", "[sparse_array_iterator_with_filter]") { create_sparse_array("test_sparse_array_it_filter"); setup_array_iterator("ATTR_INT32 > 3"); check_array_iterator(4); finalize_array_iterator(); } TEST_CASE_METHOD(ArrayIteratorFixture, "Test sparse array iterator with filter with subarray", "[sparse_array_iterator_with_filter_reset_subarray]") { create_sparse_array("test_sparse_array_it_filter_reset"); setup_array_iterator("ATTR_INT32 > 3"); check_array_iterator(4); int64_t subarray[] = { 0, 4, 0, 4 }; CHECK_RC(tiledb_array_iterator_reset_subarray(tiledb_array_iterator, subarray), TILEDB_OK); check_array_iterator(1); finalize_array_iterator(); } TEST_CASE_METHOD(ArrayIteratorFixture, "Test sparse array iterator with filter with empty value", "[sparse_array_iterator_with_filter_with_empty_value]") { create_sparse_array("test_sparse_array_it_filter_with_empty_value"); update_array_with_empty_attributes(); setup_array_iterator("ATTR_INT32 > 3"); check_array_iterator(4, true); finalize_array_iterator(); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/c_api/test_array_schema_api.cc000066400000000000000000000273551453617025200260370ustar00rootroot00000000000000/** * @file test_array_schema_api.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the C API array schema spec. */ #include "c_api_array_schema_spec.h" #include "utils.h" #include #include "catch.h" ArraySchemaTestFixture::ArraySchemaTestFixture() { // Error code int rc; // Array schema not set yet array_schema_set_ = false; // Initialize context rc = tiledb_ctx_init(&tiledb_ctx_, NULL); REQUIRE(rc == TILEDB_OK); // Create workspace rc = tiledb_workspace_create(tiledb_ctx_, WORKSPACE.c_str()); REQUIRE(rc == TILEDB_OK); // Set array name array_name_ = WORKSPACE + ARRAYNAME; } ArraySchemaTestFixture::~ArraySchemaTestFixture() { // Error code int rc; // Finalize TileDB context rc = tiledb_ctx_finalize(tiledb_ctx_); REQUIRE(rc == TILEDB_OK); // Free array schema if(array_schema_set_) { rc = tiledb_array_free_schema(&array_schema_); REQUIRE(rc == TILEDB_OK); } } /* ****************************** */ /* PUBLIC METHODS */ /* ****************************** */ int ArraySchemaTestFixture::create_dense_array(std::string array_name, int attribute_datatype, int compression_type) { // Initialization s int rc; const char* attributes[] = { "MY_ATTRIBUTE" }; const char* dimensions[] = { "X", "Y" }; int64_t domain[] = { 0, 99, 0, 1L<<62 }; int64_t tile_extents[] = { 10, 10 }; const int types[] = { attribute_datatype, TILEDB_INT64 }; const int compression[] = { compression_type, TILEDB_NO_COMPRESSION }; // Set array schema rc = tiledb_array_set_schema( // The array schema structure &array_schema_, // Array name array_name.c_str(), // Attributes attributes, // Number of attributes 1, // Capacity 1000, // Cell order TILEDB_COL_MAJOR, // Number of cell values per attribute (NULL means 1 everywhere) NULL, // Compression compression, // Compression level, NULL will get defaults NULL, // Offsets compression NULL, // Offsets compression level NULL, // Dense array 1, // Dimensions dimensions, // Number of dimensions 2, // Domain domain, // Domain length in bytes 4*sizeof(int64_t), // Tile extents (no regular tiles defined) tile_extents, // Tile extents in bytes 2*sizeof(int64_t), // Tile order (0 means ignore in sparse arrays and default in dense) 0, // Types types ); if(rc != TILEDB_OK) return TILEDB_ERR; // Remember that the array schema is set array_schema_set_ = true; // Create the array return tiledb_array_create(tiledb_ctx_, &array_schema_); } std::string path_name(const std::string& path) { size_t found = path.find_last_of("/"); if (found > 0) { found++; } return path.substr(found); } void ArraySchemaTestFixture::check_dense_array(std::string array_name) { int rc; // Load array schema from the disk TileDB_ArraySchema array_schema_disk; rc = tiledb_array_load_schema( tiledb_ctx_, array_name.c_str(), &array_schema_disk); REQUIRE(rc == TILEDB_OK); // For easy reference int64_t* domain_disk = static_cast(array_schema_disk.domain_); int64_t* domain = static_cast(array_schema_.domain_); int64_t* tile_extents_disk = static_cast(array_schema_disk.tile_extents_); int64_t* tile_extents = static_cast(array_schema_.tile_extents_); // Get real array path std::string array_name_real = fs_.real_dir(array_name); CHECK_FALSE(array_name_real == ""); // Check the last segment of the path CHECK(path_name(array_name_real) == path_name(array_name)); // Tests //absolute path isn't relevant anymore //ASSERT_STREQ(array_schema_disk.array_name_, array_name_real.c_str()); CHECK(array_schema_disk.attribute_num_ == array_schema_.attribute_num_); CHECK(array_schema_disk.dim_num_ == array_schema_.dim_num_); CHECK(array_schema_disk.capacity_ == array_schema_.capacity_); CHECK(array_schema_disk.cell_order_ == array_schema_.cell_order_); CHECK(array_schema_disk.tile_order_ == array_schema_.tile_order_); CHECK(array_schema_disk.dense_ == array_schema_.dense_); CHECK_THAT(array_schema_disk.attributes_[0], Equals(array_schema_.attributes_[0])); CHECK_THAT(array_schema_disk.dimensions_[0], Equals(array_schema_.dimensions_[0])); CHECK_THAT(array_schema_disk.dimensions_[1], Equals(array_schema_.dimensions_[1])); CHECK(domain_disk[0] == domain[0]); CHECK(domain_disk[1] == domain[1]); CHECK(domain_disk[2] == domain[2]); CHECK(domain_disk[3] == domain[3]); CHECK(array_schema_disk.compression_[0] == array_schema_.compression_[0]); CHECK(array_schema_disk.compression_[1] == array_schema_.compression_[1]); CHECK(array_schema_disk.types_[0] == array_schema_.types_[0]); CHECK(array_schema_disk.types_[1] == array_schema_.types_[1]); CHECK(tile_extents_disk[0] == tile_extents[0]); CHECK(tile_extents_disk[1] == tile_extents[1]); // Free array schema rc = tiledb_array_free_schema(&array_schema_disk); REQUIRE(rc == TILEDB_OK); } /* ****************************** */ /* TESTS */ /* ****************************** */ /** * Tests the array schema creation and retrieval. */ TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema char", "[array_schema_char]") { std::string array_name = array_name_ + "char"; // Create array int rc = create_dense_array(array_name, TILEDB_CHAR, TILEDB_NO_COMPRESSION); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema int8", "[array_schema_int8]") { std::string array_name = array_name_ + "int8"; // Create array int rc = create_dense_array(array_name, TILEDB_INT8, TILEDB_NO_COMPRESSION); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema int16", "[array_schema_int16]") { std::string array_name = array_name_ + "int16"; // Create array int rc = create_dense_array(array_name, TILEDB_INT16, TILEDB_ZSTD); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema int32", "[array_schema_int32]") { std::string array_name = array_name_ + "int32"; // Create array int rc = create_dense_array(array_name, TILEDB_INT32, TILEDB_LZ4); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema int64", "[array_schema_int64]") { std::string array_name = array_name_ + "int64"; // Create array int rc = create_dense_array(array_name, TILEDB_INT64, TILEDB_BLOSC); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema uint8", "[array_schema_uint8]") { std::string array_name = array_name_ + "uint8"; // Create array int rc = create_dense_array(array_name, TILEDB_UINT8, TILEDB_GZIP); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema uint16", "[array_schema_uint16]") { std::string array_name = array_name_ + "uint16"; // Create array int rc = create_dense_array(array_name, TILEDB_UINT16, TILEDB_ZSTD); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema uint32", "[array_schema_uint32]") { std::string array_name = array_name_ + "uint32"; // Create array int rc = create_dense_array(array_name, TILEDB_UINT32, TILEDB_LZ4); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema uint64", "[array_schema_uint64]") { std::string array_name = array_name_ + "uint64"; // Create array int rc = create_dense_array(array_name, TILEDB_UINT64, TILEDB_LZ4); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema float32", "[array_schema_float32]") { std::string array_name = array_name_ + "float32"; // Create array int rc = create_dense_array(array_name, TILEDB_FLOAT32, TILEDB_RLE); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE_METHOD(ArraySchemaTestFixture, "Test Array Schema float64", "[array_schema_float64]") { std::string array_name = array_name_ + "float64"; // Create array int rc = create_dense_array(array_name, TILEDB_FLOAT64, TILEDB_BLOSC); REQUIRE(rc == TILEDB_OK); check_dense_array(array_name); } TEST_CASE("Test array schema backward compatibility", "[compatibility]") { TileDB_CTX* tiledb_ctx; int rc = tiledb_ctx_init(&tiledb_ctx, NULL); REQUIRE(rc == TILEDB_OK); PosixFS fs; std::string array_name = std::string(TILEDB_TEST_DIR)+"/inputs/compatibility_gdb_pre100_ws/t0_1_2"; REQUIRE(fs.is_dir(array_name)); TileDB_ArraySchema array_schema; rc = tiledb_array_load_schema(tiledb_ctx, array_name.c_str(), &array_schema); REQUIRE(rc == TILEDB_OK); CHECK_THAT(array_schema.array_name_, Equals("/tmp/tmp7l5mFz/ws/t0_1_2")); CHECK(array_schema.attribute_num_ == 22); CHECK(array_schema.capacity_ == 3); CHECK(array_schema.cell_order_ == 1); CHECK(array_schema.dim_num_ == 2); CHECK(array_schema.compression_[0] == 1); CHECK(tiledb_array_free_schema(&array_schema) == TILEDB_OK); CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } TEST_CASE("Test array schema backward compatibility 2", "[compatibility_2]") { TileDB_CTX* tiledb_ctx; int rc = tiledb_ctx_init(&tiledb_ctx, NULL); REQUIRE(rc == TILEDB_OK); PosixFS fs; std::string array_name = std::string(TILEDB_TEST_DIR)+"/inputs/compatibility_pre2_ws/my_workspace/sparse_arrays/my_array_B"; REQUIRE(fs.is_dir(array_name)); TileDB_ArraySchema array_schema; rc = tiledb_array_load_schema(tiledb_ctx, array_name.c_str(), &array_schema); REQUIRE(rc == TILEDB_OK); CHECK(std::string(array_schema.array_name_).find(std::string("my_array_B")) != std::string::npos); CHECK(array_schema.attribute_num_ == 3); CHECK(array_schema.capacity_ == 2); CHECK(array_schema.cell_order_ == 0); CHECK(array_schema.dim_num_ == 2); CHECK(array_schema.compression_[0] == TILEDB_GZIP); CHECK(array_schema.compression_level_[1] == -1); CHECK(array_schema.offsets_compression_[0] == 0); CHECK(array_schema.offsets_compression_[1] == TILEDB_GZIP); CHECK(array_schema.offsets_compression_[2] == 0); CHECK(array_schema.offsets_compression_level_[1] == -1); CHECK(tiledb_array_free_schema(&array_schema) == TILEDB_OK); CHECK(tiledb_ctx_finalize(tiledb_ctx) == TILEDB_OK); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/c_api/test_dense_array_api.cc000066400000000000000000000760351453617025200256740ustar00rootroot00000000000000/** * @file c_api_dense_array_spec.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2019 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests of C API for dense array operations. */ #include "c_api_dense_array_spec.h" #include "progress_bar.h" #include "storage_posixfs.h" #include #include #include #include #include #include #include "catch.h" DenseArrayTestFixture::DenseArrayTestFixture() { // Reset the random number generator srand(0); // Error code int rc; // Initialize context rc = tiledb_ctx_init(&tiledb_ctx_, NULL); CHECK_RC(rc, TILEDB_OK); // Create workspace rc = tiledb_workspace_create(tiledb_ctx_, WORKSPACE.c_str()); CHECK_RC(rc, TILEDB_OK); } DenseArrayTestFixture::~DenseArrayTestFixture() { // Error code int rc; // Finalize TileDB context rc = tiledb_ctx_finalize(tiledb_ctx_); CHECK_RC(rc, TILEDB_OK); } /* ****************************** */ /* PUBLIC METHODS */ /* ****************************** */ bool DenseArrayTestFixture::check_buffer_after_updates( const int* buffer_before, const int* buffer_after, const int* buffer_updates_a1, const int64_t *buffer_updates_coords, const int64_t domain_size_0, const int64_t domain_size_1, const int64_t update_num) { // Initializations int l,r; int64_t cell_num = domain_size_0*domain_size_1; // Check the contents of the buffers cell by cell for(int64_t i = 0; i < cell_num; ++i) { l = buffer_before[i]; r = buffer_after[i]; // If they are not the same, check if it is due to an update if(l!=r) { bool found = false; for(int64_t k = 0; k < update_num; ++k) { // The difference is due to an update if(r == buffer_updates_a1[k] && (l/domain_size_1) == buffer_updates_coords[2*k] && (l%domain_size_1) == buffer_updates_coords[2*k+1]) { found = true; break; } } // The difference is not due to an update if(!found) return false; } } // Success return true; } template void* create_buffer_for_domain(int32_t domain_size, T** buffer, size_t* bytes) { T *typed_buffer= *buffer; typed_buffer = new T[domain_size]; *bytes = domain_size*sizeof(T); for(int32_t i = 0; i < domain_size; ++i) typed_buffer[i] = (T)i; return reinterpret_cast(typed_buffer); } template void clear_buffer_for_domain(int32_t domain_size, T* buffer) { for(int32_t i = 0; i < domain_size; ++i) buffer[i] = (T)0; } template void validate_and_cleanup_buffer_for_domain(int32_t domain_size, T* buffer) { for(int32_t i = 0; i < domain_size; ++i) CHECK(buffer[i] == (T)i); delete buffer; } int DenseArrayTestFixture::create_dense_array_1D( const int attribute_type, const int32_t tile_extent, const int32_t domain_lo, const int32_t domain_hi, const int cell_order, const int tile_order) { // Error code int rc; // Setup array const int attribute_num = 1; const char* attributes[] = { "MY_ATTRIBUTE" }; const char* dimensions[] = { "X"}; int32_t domain[] = { domain_lo, domain_hi }; int32_t tile_extents[] = { tile_extent }; const int types[] = { attribute_type, TILEDB_INT32 }; int compression[] = { TILEDB_NO_COMPRESSION, TILEDB_NO_COMPRESSION }; int compression_level[] = { 0, 0 }; const int dense = 1; // Set the array schema rc = tiledb_array_set_schema( &array_schema_, array_name_.c_str(), attributes, attribute_num, 0, cell_order, NULL, compression, compression_level, NULL, // offsets compression NULL, // offsets compression level dense, dimensions, 1, domain, 2*sizeof(int32_t), tile_extents, sizeof(int32_t), tile_order, types); if(rc != TILEDB_OK) return TILEDB_ERR; // Create the array rc = tiledb_array_create(tiledb_ctx_, &array_schema_); if(rc != TILEDB_OK) return TILEDB_ERR; // Free array schema rc = tiledb_array_free_schema(&array_schema_); if(rc != TILEDB_OK) return TILEDB_ERR; int32_t domain_size = domain_hi-domain_lo+1; size_t nbytes = domain_size; void *buffer = nullptr; if (attribute_type == TILEDB_CHAR) { char *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_INT8) { int8_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_INT16) { int16_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_INT32) { int32_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_INT64) { int64_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_UINT8) { uint8_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_UINT16) { uint16_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_UINT32) { uint32_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_UINT64) { uint64_t *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_FLOAT32) { float *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } else if (attribute_type == TILEDB_FLOAT64) { double *typed_buffer = reinterpret_cast(buffer); buffer = create_buffer_for_domain(domain_size, &typed_buffer, &nbytes); } CHECK(buffer != nullptr); std::vector buffers; std::vector buffer_sizes; buffers.push_back(buffer); buffer_sizes.push_back(nbytes); // Intialize array TileDB_Array* tiledb_array; rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_WRITE, NULL, NULL, 0); if(rc != TILEDB_OK) return TILEDB_ERR; // Write array rc = tiledb_array_write(tiledb_array, const_cast(buffers.data()), buffer_sizes.data()); if(rc != TILEDB_OK) return TILEDB_ERR; // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) return TILEDB_ERR; // Clear buffer if (attribute_type == TILEDB_CHAR) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT8) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT16) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT32) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT64) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT8) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT16) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT32) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT64) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_FLOAT32) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_FLOAT64) { clear_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } // Read array rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_READ, NULL, NULL, 0); if(rc != TILEDB_OK) return TILEDB_ERR; rc = tiledb_array_read(tiledb_array, buffers.data(), buffer_sizes.data()); if(rc != TILEDB_OK) return TILEDB_ERR; // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) return TILEDB_ERR; // Check buffer if (attribute_type == TILEDB_CHAR) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT8) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT16) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT32) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_INT64) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT8) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT16) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT32) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_UINT64) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_FLOAT32) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } else if (attribute_type == TILEDB_FLOAT64) { validate_and_cleanup_buffer_for_domain(domain_size, reinterpret_cast(buffer)); } return TILEDB_OK; } int DenseArrayTestFixture::create_dense_array_2D( const int64_t tile_extent_0, const int64_t tile_extent_1, const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int64_t capacity, const bool enable_compression, const int cell_order, const int tile_order) { // Error code int rc; // Prepare and set the array schema object and data structures const int attribute_num = 1; const char* attributes[] = { "ATTR_INT32" }; const char* dimensions[] = { "X", "Y" }; int64_t domain[] = { domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi }; int64_t tile_extents[] = { tile_extent_0, tile_extent_1 }; const int types[] = { TILEDB_INT32, TILEDB_INT64 }; int compression[2]; const int dense = 1; if(!enable_compression) { compression[0] = TILEDB_NO_COMPRESSION; compression[1] = TILEDB_NO_COMPRESSION; } else { compression[0] = TILEDB_GZIP; compression[1] = TILEDB_GZIP; } // Set the array schema rc = tiledb_array_set_schema( &array_schema_, array_name_.c_str(), attributes, attribute_num, capacity, cell_order, NULL, compression, NULL, NULL, // offsets compression NULL, // offsets compression level dense, dimensions, 2, domain, 4*sizeof(int64_t), tile_extents, 2*sizeof(int64_t), tile_order, types); if(rc != TILEDB_OK) return TILEDB_ERR; // Create the array rc = tiledb_array_create(tiledb_ctx_, &array_schema_); if(rc != TILEDB_OK) return TILEDB_ERR; // Free array schema rc = tiledb_array_free_schema(&array_schema_); if(rc != TILEDB_OK) return TILEDB_ERR; // Success return TILEDB_OK; } int* DenseArrayTestFixture::generate_1D_int_buffer( const int64_t domain_size_0, const int64_t domain_size_1) { // Create buffer int *buffer = new int[domain_size_0 * domain_size_1]; // Populate buffer for(int64_t i = 0; i < domain_size_0; ++i) for(int64_t j = 0; j < domain_size_1; ++j) buffer[i*domain_size_1+j] = i * domain_size_1 + j; // Return return buffer; } int** DenseArrayTestFixture::generate_2D_buffer( const int64_t domain_size_0, const int64_t domain_size_1) { // Create buffer int **buffer = new int*[domain_size_0]; // Populate buffer for(int64_t i = 0; i < domain_size_0; ++i) { buffer[i] = new int [domain_size_1]; for(int64_t j = 0; j < domain_size_1; ++j) { buffer[i][j] = i * domain_size_1 + j; } } // Return return buffer; } int* DenseArrayTestFixture::read_dense_array_2D( const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int read_mode) { // Error code int rc; // Initialize a subarray const int64_t subarray[] = { domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi }; // Subset over a specific attribute const char* attributes[] = { "ATTR_INT32" }; // Initialize the array in the input mode TileDB_Array* tiledb_array; rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), read_mode, subarray, attributes, 1); if(rc != TILEDB_OK) return NULL; // Prepare the buffers that will store the result int64_t domain_size_0 = domain_0_hi - domain_0_lo + 1; int64_t domain_size_1 = domain_1_hi - domain_1_lo + 1; int64_t cell_num = domain_size_0 * domain_size_1; int* buffer_a1 = new int[cell_num]; void* buffers[] = { buffer_a1 }; size_t buffer_size_a1 = cell_num * sizeof(int); size_t buffer_sizes[] = { buffer_size_a1 }; // Read from array rc = tiledb_array_read(tiledb_array, buffers, buffer_sizes); if(rc != TILEDB_OK) { delete [] buffer_a1; return NULL; } // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) { delete [] buffer_a1; return NULL; } // Success - return the created buffer return buffer_a1; } void DenseArrayTestFixture::set_array_name(const char *name) { array_name_ = WORKSPACE + name; } int DenseArrayTestFixture::update_dense_array_2D( const int64_t domain_size_0, const int64_t domain_size_1, int64_t update_num, int seed, void** buffers, const size_t* buffer_sizes) { // Error code int rc; // For easy reference int* buffer_a1 = (int*) buffers[0]; int64_t* buffer_coords = (int64_t*) buffers[1]; // Specify attributes to be written const char* attributes[] = { "ATTR_INT32", TILEDB_COORDS }; // Initialize the array TileDB_Array* tiledb_array; rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_WRITE_UNSORTED, NULL, attributes, 2); if(rc != TILEDB_OK) return TILEDB_ERR; // Populate buffers with random updates srand(seed); int64_t x, y, v; int64_t coords_index = 0L; std::map my_map; std::map::iterator it; my_map.clear(); for(int64_t i = 0; i < update_num; ++i) { std::ostringstream rand_stream; do { std::ostringstream rand_stream; x = rand() % domain_size_0; y = rand() % domain_size_1; v = rand(); rand_stream << x << "," << y; it = my_map.find(rand_stream.str()); } while (it != my_map.end()); rand_stream << x << "," << y; my_map[rand_stream.str()] = v; buffer_coords[coords_index++] = x; buffer_coords[coords_index++] = y; buffer_a1[i] = v; } // Write to array rc = tiledb_array_write( tiledb_array, (const void**) buffers, buffer_sizes); if(rc != TILEDB_OK) return TILEDB_ERR; // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) return TILEDB_ERR; // Success return TILEDB_OK; } int DenseArrayTestFixture::write_dense_array_by_tiles( const int64_t domain_size_0, const int64_t domain_size_1, const int64_t tile_extent_0, const int64_t tile_extent_1) { // Error code int rc; // Initialize the array TileDB_Array* tiledb_array; rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_WRITE, NULL, NULL, 0); if(rc != TILEDB_OK) return TILEDB_ERR; // Other initializations int** buffer = generate_2D_buffer(domain_size_0, domain_size_1); int64_t cell_num_in_tile = tile_extent_0 * tile_extent_1; int* buffer_a1 = new int[cell_num_in_tile]; for(int64_t i = 0; i < cell_num_in_tile; ++i) buffer_a1[i] = 0; const void* buffers[2]; buffers[0] = buffer_a1; size_t buffer_sizes[2]; int64_t index = 0L; size_t buffer_size = 0L; // Populate and write tile by tile for(int64_t i = 0; i < domain_size_0; i += tile_extent_0) { for(int64_t j = 0; j < domain_size_1; j += tile_extent_1) { int64_t tile_rows = ((i + tile_extent_0) < domain_size_0) ? tile_extent_0 : (domain_size_0 - i); int64_t tile_cols = ((j + tile_extent_1) < domain_size_1) ? tile_extent_1 : (domain_size_1 - j); int64_t k,l; for(k = 0; k < tile_rows; ++k) { for(l = 0; l < tile_cols; ++l) { index = uint64_t(k * tile_cols + l); buffer_a1[index] = buffer[uint64_t(i + k)][uint64_t(j + l)]; } } buffer_size = k*l*sizeof(int); buffer_sizes[0] = { buffer_size }; rc = tiledb_array_write(tiledb_array, buffers, buffer_sizes); if(rc != TILEDB_OK) { tiledb_array_finalize(tiledb_array); for(int64_t i=0; iload(1.0/iter_num); } // Delete progress bar delete progress_bar; } /** * Tests random 2D subarray writes. */ TEST_CASE_METHOD(DenseArrayTestFixture, "Test random 2D subarray writes", "[test_random_dense_sorted_writes]") { char *disable_file_locking_env = strdup("TILEDB_DISABLE_FILE_LOCKING=True"); CHECK(putenv(disable_file_locking_env) == 0); // Error code int rc; // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; int64_t tile_extent_0 = 10; int64_t tile_extent_1 = 10; int64_t domain_0_lo = 0; int64_t domain_0_hi = domain_size_0-1; int64_t domain_1_lo = 0; int64_t domain_1_hi = domain_size_1-1; int64_t capacity = 0; // 0 means use default capacity int cell_order = TILEDB_ROW_MAJOR; int tile_order = TILEDB_ROW_MAJOR; int iter_num = 10; // Set array name set_array_name("dense_test_100x100_10x10"); // Create a progress bar ProgressBar* progress_bar = new ProgressBar(); // Create a dense integer array rc = create_dense_array_2D( tile_extent_0, tile_extent_1, domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi, capacity, false, cell_order, tile_order); CHECK_RC(rc, TILEDB_OK); PosixFS fs; CHECK(fs.is_dir(WORKSPACE+"/dense_test_100x100_10x10")); CHECK(!fs.is_file(WORKSPACE+"/dense_test_100x100_10x10/.consolidation_lock")); // Write random subarray, then read it back and check int64_t d0[2], d1[2]; for(int i = 0; i < iter_num; ++i) { // Create subarray d0[0] = rand() % domain_size_0; d1[0] = rand() % domain_size_1; d0[1] = d0[0] + rand() % (domain_size_0 - d0[0]); d1[1] = d1[0] + rand() % (domain_size_1 - d1[0]); int64_t subarray[] = { d0[0], d0[1], d1[0], d1[1] }; // Prepare buffers int64_t subarray_length[2] = { d0[1] - d0[0] + 1, d1[1] - d1[0] + 1 }; int64_t cell_num_in_subarray = subarray_length[0] * subarray_length[1]; int *buffer = new int[cell_num_in_subarray]; int64_t index = 0; size_t buffer_size = cell_num_in_subarray*sizeof(int); size_t buffer_sizes[] = { buffer_size }; for(int64_t r = 0; r < subarray_length[0]; ++r) for(int64_t c = 0; c < subarray_length[1]; ++c) buffer[index++] = - (rand() % 999999); // Write 2D subarray rc = write_dense_subarray_2D( subarray, TILEDB_ARRAY_WRITE_SORTED_ROW, buffer, buffer_sizes); CHECK_RC(rc, TILEDB_OK); // Read back the same subarray int* read_buffer = read_dense_array_2D( subarray[0], subarray[1], subarray[2], subarray[3], TILEDB_ARRAY_READ_SORTED_ROW); REQUIRE(read_buffer != NULL); // Check the two buffers for(index = 0; index < cell_num_in_subarray; ++index) CHECK(buffer[index] == read_buffer[index]); // Clean up delete [] buffer; delete [] read_buffer; // Update progress bar progress_bar->load(1.0/iter_num); } // Delete progress bar delete progress_bar; free(disable_file_locking_env); disable_file_locking_env = strdup("TILEDB_DISABLE_FILE_LOCKING=False"); CHECK(putenv(disable_file_locking_env) == 0); free(disable_file_locking_env); } /** * Test random updates in a 2D dense array. */ TEST_CASE_METHOD(DenseArrayTestFixture, " Test random updates in a 2D dense array", "[test_random_dense_updates]") { // Error code int rc; // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; int64_t tile_extent_0 = 10; int64_t tile_extent_1 = 10; int64_t domain_0_lo = 0; int64_t domain_0_hi = domain_size_0-1; int64_t domain_1_lo = 0; int64_t domain_1_hi = domain_size_1-1; int64_t capacity = 0; // 0 means use default capacity int cell_order = TILEDB_ROW_MAJOR; int tile_order = TILEDB_ROW_MAJOR; int64_t update_num = 100; int seed = 7; // Set array name set_array_name("dense_test_100x100_10x10"); // Create a dense integer array rc = create_dense_array_2D( tile_extent_0, tile_extent_1, domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi, capacity, false, cell_order, tile_order); CHECK_RC(rc, TILEDB_OK); // Write array cells with value = row id * COLUMNS + col id // to disk tile by tile rc = write_dense_array_by_tiles( domain_size_0, domain_size_1, tile_extent_0, tile_extent_1); CHECK_RC(rc, TILEDB_OK); // Read the entire array back to memory int *before_update = read_dense_array_2D( domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi, TILEDB_ARRAY_READ); REQUIRE(before_update != NULL); // Prepare random updates int *buffer_a1 = new int[update_num]; int64_t *buffer_coords = new int64_t[2*update_num]; void* buffers[] = { buffer_a1, buffer_coords}; size_t buffer_sizes[2]; buffer_sizes[0] = update_num*sizeof(int); buffer_sizes[1] = 2*update_num*sizeof(int64_t); rc = update_dense_array_2D( domain_size_0, domain_size_1, update_num, seed, buffers, buffer_sizes); CHECK_RC(rc, TILEDB_OK); // Read the entire array back to memory after update int *after_update = read_dense_array_2D( domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi, TILEDB_ARRAY_READ); REQUIRE(after_update != NULL); // Compare array before and after bool success = check_buffer_after_updates( before_update, after_update, buffer_a1, buffer_coords, domain_size_0, domain_size_1, update_num); CHECK(success); // Clean up delete [] before_update; delete [] after_update; delete [] buffer_a1; delete [] buffer_coords; } genomicsdb-0.0~git20231212.9d7ddd0/test/src/c_api/test_sparse_array_api.cc000066400000000000000000000554011453617025200260650ustar00rootroot00000000000000/** * @file c_api_sparse_array_spec.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2020, 2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests of C API for sparse array operations. */ #include "catch.h" #include "c_api_sparse_array_spec.h" #include "progress_bar.h" #include "storage_manager.h" #include "storage_posixfs.h" #include "utils.h" #include #include #include #include #include #include template class ArrayTestFixture1D : TempDir { public: const std::string WORKSPACE = get_temp_dir() + "/array_test_fixture_ws_1D/"; std::string array_name_; TileDB_ArraySchema array_schema_; TileDB_CTX* tiledb_ctx_; std::shared_ptr> buffer_; size_t num_elements_ = 10; std::shared_ptr> buffer_coords_; size_t buffer_coords_size_; std::vector buffers_; std::vector buffer_sizes_; ArrayTestFixture1D() { CHECK_RC(tiledb_ctx_init(&tiledb_ctx_, NULL), TILEDB_OK); CHECK_RC(tiledb_workspace_create(tiledb_ctx_, WORKSPACE.c_str()), TILEDB_OK); } ~ArrayTestFixture1D() { // Finalize TileDB context int rc = tiledb_ctx_finalize(tiledb_ctx_); CHECK_RC(rc, TILEDB_OK); } // Create a buffer to store values of given Type // Returns a buffer and size to calling client void create_buffer(size_t size, T** buffer, size_t* bytes) { buffer_ = std::make_shared>(size); // Set some arbitrary values for(auto i = 0u; i < size; ++i) { buffer_.get()->data()[i] = (T)i; } *bytes = (buffer_.get()->size())*sizeof(T); *buffer = &buffer_.get()->data()[0]; } void clear_buffer() { for(auto i = 0u; i < buffer_.get()->size(); i++) { buffer_.get()->data()[i] = (T)0; } for(auto i = 0u; i < buffer_coords_.get()->size(); i++) { buffer_coords_.get()->data()[i] = 0; } } void validate_buffer() { for(auto i = 0u; i < buffer_coords_.get()->size(); i++) { CHECK(buffer_.get()->data()[i] == (T)(buffer_coords_.get()->data()[i])); } } int create_array(const int32_t tile_extent, const int32_t domain_lo, const int32_t domain_hi, const int cell_order, const int tile_order) { // Error code int rc; int attribute_type = TILEDB_CHAR; if (typeid(T) == typeid(uint8_t)) { attribute_type = TILEDB_UINT8; } else if (typeid(T) == typeid(uint16_t)) { attribute_type = TILEDB_UINT16; } else if (typeid(T) == typeid(uint32_t)) { attribute_type = TILEDB_UINT32; } else if (typeid(T) == typeid(uint64_t)) { attribute_type = TILEDB_UINT64; } else if (typeid(T) == typeid(int8_t)) { attribute_type = TILEDB_INT8; } else if (typeid(T) == typeid(int16_t)) { attribute_type = TILEDB_INT16; } else if (typeid(T) == typeid(int) || typeid(T) == typeid(int32_t)) { attribute_type = TILEDB_INT32; } else if (typeid(T) == typeid(int64_t)) { attribute_type = TILEDB_INT64; } else if (typeid(T) == typeid(float)) { attribute_type = TILEDB_FLOAT32; } else if (typeid(T) == typeid(double)) { attribute_type = TILEDB_FLOAT64; } // Setup array const int attribute_num = 1; const char* attributes[] = { "MY_ATTRIBUTE" }; const char* dimensions[] = { "X"}; int32_t domain[] = { domain_lo, domain_hi }; int32_t tile_extents[] = { tile_extent }; const int types[] = { attribute_type, TILEDB_INT32 }; int compression[] = { TILEDB_NO_COMPRESSION, TILEDB_NO_COMPRESSION}; int compression_level[] = { 0, 0 }; // Set the array schema rc = tiledb_array_set_schema( &array_schema_, array_name_.c_str(), attributes, attribute_num, 0, cell_order, NULL, compression, compression_level, NULL, // Offsets compression NULL, // Offsets compression level 0, // Sparse dimensions, 1, domain, 2*sizeof(int32_t), tile_extents, sizeof(int32_t), tile_order, types); CHECK_RC(rc, TILEDB_OK); // Create the array rc = tiledb_array_create(tiledb_ctx_, &array_schema_); CHECK_RC(rc, TILEDB_OK); // Free array schema rc = tiledb_array_free_schema(&array_schema_); CHECK_RC(rc, TILEDB_OK); buffer_coords_ = std::make_shared>(num_elements_); for(auto i = 0u; i < num_elements_; ++i) { buffer_coords_.get()->data()[i] = (int32_t)i; } size_t buffer_coords_size = num_elements_ * sizeof(int32_t); T *buffer = nullptr; size_t buffer_bytes = 0; create_buffer(num_elements_, &buffer, &buffer_bytes); CHECK(buffer != nullptr); buffers_.push_back(buffer); buffer_sizes_.push_back(buffer_bytes); buffers_.push_back(&buffer_coords_.get()->data()[0]); buffer_sizes_.push_back(buffer_coords_size); return TILEDB_OK; } int write_array() { // Intialize array TileDB_Array* tiledb_array; int rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_WRITE, NULL, NULL, 0); CHECK_RC(rc, TILEDB_OK); // Write array rc = tiledb_array_write(tiledb_array, const_cast(buffers_.data()), buffer_sizes_.data()); CHECK_RC(rc, TILEDB_OK); // Finalize the array rc = tiledb_array_finalize(tiledb_array); CHECK_RC(rc, TILEDB_OK); return TILEDB_OK; } int read_array() { clear_buffer(); // Initialize array TileDB_Array* tiledb_array; int rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_READ, NULL, NULL, 0); CHECK_RC(rc, TILEDB_OK); // Read array rc = tiledb_array_read(tiledb_array, buffers_.data(), buffer_sizes_.data()); CHECK_RC(rc, TILEDB_OK); CHECK(buffer_sizes_.data()[0] == num_elements_*sizeof(T)); CHECK(buffer_sizes_.data()[1] == num_elements_*sizeof(int32_t)); // Check no overflow // CHECK_RC(tiledb_array_overflow(tiledb_array, 0), TILEDB_OK); // Finalize the array rc = tiledb_array_finalize(tiledb_array); CHECK_RC(rc, TILEDB_OK); validate_buffer(); return TILEDB_OK; } int consolidate_array(size_t segment_size=0, int batch_size=0) { // Consolidate array int rc; if (batch_size) { rc = tiledb_array_consolidate(tiledb_ctx_, array_name_.c_str(), segment_size, batch_size); } else if (segment_size) { rc = tiledb_array_consolidate(tiledb_ctx_, array_name_.c_str(), segment_size); } else { rc = tiledb_array_consolidate(tiledb_ctx_, array_name_.c_str()); } CHECK_RC(rc, TILEDB_OK); return TILEDB_OK; } void set_array_name(const char *name) { array_name_ = WORKSPACE + name; } }; #define AF ArrayTestFixture1D TEMPLATE_TEST_CASE_METHOD(ArrayTestFixture1D, "Test Sparse 1D Array", "[test_sparse_1D_array]", char, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("row_major") { AF::set_array_name("sparse_test_1D_row"); CHECK_RC(AF::create_array(5, 0, 99, TILEDB_ROW_MAJOR/*cell_order*/, TILEDB_ROW_MAJOR/*tile_order*/), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); CHECK_RC(AF::consolidate_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::consolidate_array(200, 2), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); } SECTION("col_major") { AF::set_array_name("sparse_test_1D_col"); CHECK_RC(AF::create_array(10, 0, 990, TILEDB_COL_MAJOR, TILEDB_COL_MAJOR), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); CHECK_RC(AF::consolidate_array(200, 1), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); } SECTION("col_mixed") { AF::set_array_name("sparse_test_1D_mixed"); CHECK_RC(AF::create_array(100, 0, 990, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); } SECTION("col_mixed_1") { AF::set_array_name("sparse_test_1D_mixed_1"); CHECK_RC(AF::create_array(200, 0, 990, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR), TILEDB_OK); CHECK_RC(AF::write_array(), TILEDB_OK); CHECK_RC(AF::read_array(), TILEDB_OK); } } SparseArrayTestFixture::SparseArrayTestFixture() { // Error code int rc; // Initialize context rc = tiledb_ctx_init(&tiledb_ctx_, NULL); CHECK_RC(rc, TILEDB_OK); // Create workspace rc = tiledb_workspace_create(tiledb_ctx_, WORKSPACE.c_str()); CHECK_RC(rc, TILEDB_OK); } SparseArrayTestFixture::~SparseArrayTestFixture() { // Error code int rc; // Finalize TileDB context rc = tiledb_ctx_finalize(tiledb_ctx_); CHECK_RC(rc, TILEDB_OK); } /* ****************************** */ /* PUBLIC METHODS */ /* ****************************** */ int SparseArrayTestFixture::create_sparse_array_2D( const int64_t tile_extent_0, const int64_t tile_extent_1, const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int64_t capacity, const bool enable_compression, const int cell_order, const int tile_order) { // Error code int rc; // Prepare and set the array schema object and data structures const int attribute_num = 1; const char* attributes[] = { "ATTR_INT32" }; const char* dimensions[] = { "X", "Y" }; int64_t domain[] = { domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi }; int64_t tile_extents[] = { tile_extent_0, tile_extent_1 }; const int types[] = { TILEDB_INT32, TILEDB_INT64 }; int compression[2]; const int dense = 0; if(!enable_compression) { compression[0] = TILEDB_NO_COMPRESSION; compression[1] = TILEDB_NO_COMPRESSION; } else { compression[0] = TILEDB_GZIP; compression[1] = TILEDB_GZIP; } // Set the array schema rc = tiledb_array_set_schema( &array_schema_, array_name_.c_str(), attributes, attribute_num, capacity, cell_order, NULL, compression, NULL, NULL, // Offsets compression NULL, // Offsets compression level dense, dimensions, 2, domain, 4*sizeof(int64_t), tile_extents, 2*sizeof(int64_t), tile_order, types); if(rc != TILEDB_OK) return TILEDB_ERR; // Create the array rc = tiledb_array_create(tiledb_ctx_, &array_schema_); if(rc != TILEDB_OK) return TILEDB_ERR; // Free array schema rc = tiledb_array_free_schema(&array_schema_); if(rc != TILEDB_OK) return TILEDB_ERR; // Success return TILEDB_OK; } int* SparseArrayTestFixture::read_sparse_array_2D( const int64_t domain_0_lo, const int64_t domain_0_hi, const int64_t domain_1_lo, const int64_t domain_1_hi, const int read_mode, const ssize_t expected_num_cells) { // Error code int rc; // Initialize a subarray const int64_t subarray[] = { domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi }; // Subset over a specific attribute const char* attributes[] = { "ATTR_INT32" }; // Initialize the array in the input mode TileDB_Array* tiledb_array; rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), read_mode, subarray, attributes, 1); if(rc != TILEDB_OK) return NULL; // Prepare the buffers that will store the result int64_t domain_size_0 = domain_0_hi - domain_0_lo + 1; int64_t domain_size_1 = domain_1_hi - domain_1_lo + 1; int64_t cell_num = domain_size_0 * domain_size_1; int* buffer_a1 = new int[cell_num]; assert(buffer_a1); void* buffers[] = { buffer_a1 }; size_t buffer_size_a1 = cell_num * sizeof(int); size_t buffer_sizes[] = { buffer_size_a1 }; // Read from array rc = tiledb_array_read(tiledb_array, buffers, buffer_sizes); if(rc != TILEDB_OK) { tiledb_array_finalize(tiledb_array); return NULL; } // Check buffer sizes if (expected_num_cells >= 0) { CHECK(buffer_sizes[0] == (size_t)expected_num_cells*sizeof(int)); } else { CHECK(buffer_sizes[0] == (size_t)cell_num*sizeof(int)); } // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) return NULL; // Success - return the created buffer return buffer_a1; } void SparseArrayTestFixture::set_array_name(const char *name) { array_name_ = WORKSPACE + name; } int SparseArrayTestFixture::write_sparse_array_unsorted_2D( const int64_t domain_size_0, const int64_t domain_size_1) { // Error code int rc; // Generate random attribute values and coordinates for sparse write int64_t cell_num = domain_size_0*domain_size_1; int* buffer_a1 = new int[cell_num]; int64_t* buffer_coords = new int64_t[2*cell_num]; int64_t coords_index = 0L; for (int64_t i = 0; i < domain_size_0; ++i) { for (int64_t j = 0; j < domain_size_1; ++j) { buffer_a1[i*domain_size_1+j] = i*domain_size_1+j; buffer_coords[2*coords_index] = i; buffer_coords[2*coords_index+1] = j; coords_index++; } } // Initialize the array TileDB_Array* tiledb_array; rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_WRITE_UNSORTED, NULL, NULL, 0); if(rc != TILEDB_OK) return TILEDB_ERR; // Write to array const void* buffers[] = { buffer_a1, buffer_coords }; size_t buffer_sizes[2]; buffer_sizes[0] = cell_num*sizeof(int); buffer_sizes[1] = 2*cell_num*sizeof(int64_t); rc = tiledb_array_write(tiledb_array, buffers, buffer_sizes); if(rc != TILEDB_OK) return TILEDB_ERR; // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) return TILEDB_ERR; // Clean up delete [] buffer_a1; delete [] buffer_coords; // Success return TILEDB_OK; } /** * Test is to randomly read subregions of the array and * check with corresponding value set by row_id*dim1+col_id * Top left corner is always 4,4 * Test runs through 10 iterations to choose random * width and height of the subregions */ TEST_CASE_METHOD(SparseArrayTestFixture, "Test random read subregions", "[test_random_sparse_sorted_reads]") { // Error code int rc; // Parameters used in this test int64_t domain_size_0 = 5000; int64_t domain_size_1 = 1000; int64_t tile_extent_0 = 100; int64_t tile_extent_1 = 100; int64_t domain_0_lo = 0; int64_t domain_0_hi = domain_size_0-1; int64_t domain_1_lo = 0; int64_t domain_1_hi = domain_size_1-1; int64_t capacity = 0; // 0 means use default capacity int cell_order = TILEDB_ROW_MAJOR; int tile_order = TILEDB_ROW_MAJOR; int iter_num = 10; // Set array name set_array_name("sparse_test_5000x1000_100x100"); // Create a progress bar ProgressBar* progress_bar = new ProgressBar(); // Create a dense integer array rc = create_sparse_array_2D( tile_extent_0, tile_extent_1, domain_0_lo, domain_0_hi, domain_1_lo, domain_1_hi, capacity, false, cell_order, tile_order); CHECK_RC(rc, TILEDB_OK); // Write array cells with value = row id * COLUMNS + col id // to disk rc = write_sparse_array_unsorted_2D(domain_size_0, domain_size_1); CHECK_RC(rc, TILEDB_OK); // Test random subarrays and check with corresponding value set by // row_id*dim1+col_id. Top left corner is always 4,4. int64_t d0_lo = 4; int64_t d0_hi = 0; int64_t d1_lo = 4; int64_t d1_hi = 0; int64_t height = 0, width = 0; for(int iter = 0; iter < iter_num; ++iter) { height = rand() % (domain_size_0 - d0_lo); width = rand() % (domain_size_1 - d1_lo); d0_hi = d0_lo + height; d1_hi = d1_lo + width; int64_t index = 0; // Read subarray int *buffer = read_sparse_array_2D( d0_lo, d0_hi, d1_lo, d1_hi, TILEDB_ARRAY_READ_SORTED_ROW); REQUIRE(buffer != NULL); // Check for(int64_t i = d0_lo; i <= d0_hi; ++i) { for(int64_t j = d1_lo; j <= d1_hi; ++j) { CHECK(buffer[index] == i*domain_size_1+j); if (buffer[index] != (i*domain_size_1+j)) { std::cout << "mismatch: " << i << "," << j << "=" << buffer[index] << "!=" << ((i*domain_size_1+j)) << "\n"; return; } ++index; } } // Clean up delete [] buffer; // Update progress bar progress_bar->load(1.0/iter_num); } // Delete progress bar delete progress_bar; } class SparseArrayEnvTestFixture : SparseArrayTestFixture { public: SparseArrayTestFixture *test_fixture; SparseArrayEnvTestFixture() { test_fixture = NULL; } ~SparseArrayEnvTestFixture() { delete test_fixture; } void set_disable_file_locking() { CHECK(setenv("TILEDB_DISABLE_FILE_LOCKING", "1", 1) == 0); CHECK(is_env_set("TILEDB_DISABLE_FILE_LOCKING")); PosixFS fs; REQUIRE(!fs.locking_support()); } void unset_disable_file_locking() { unsetenv("TILEDB_DISABLE_FILE_LOCKING"); CHECK(!is_env_set("TILEDB_DISABLE_FILE_LOCKING")); PosixFS fs; REQUIRE(fs.locking_support()); } void set_keep_write_file_handles_open() { CHECK(setenv("TILEDB_KEEP_FILE_HANDLES_OPEN", "1", 1) == 0); CHECK(is_env_set("TILEDB_KEEP_FILE_HANDLES_OPEN")); PosixFS fs; REQUIRE(fs.keep_write_file_handles_open()); } void unset_keep_write_file_handles_open() { unsetenv("TILEDB_KEEP_FILE_HANDLES_OPEN"); CHECK(!is_env_set("KEEP_FILE_HANDLES_OPEN")); PosixFS fs; REQUIRE(!fs.keep_write_file_handles_open()); } int write_array() { // Set array name set_array_name("sparse_test_disable_file_locking_env"); CHECK_RC(create_sparse_array_2D(4, 4, 0, 15, 0, 15, 0, false, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR), TILEDB_OK); // Write array cells with value = row id * COLUMNS + col id // to disk CHECK_RC(write_sparse_array_unsorted_2D(16, 16), TILEDB_OK); return 0; } int read_array() { int *contents = read_sparse_array_2D(0, 4, 0, 4, TILEDB_ARRAY_READ); CHECK(contents != NULL); delete [] contents; return 0; } bool consolidation_file_lock_exists() { PosixFS fs; return fs.is_file(array_name_+"/"+TILEDB_SM_CONSOLIDATION_FILELOCK_NAME); } }; TEST_CASE_METHOD(SparseArrayEnvTestFixture, "Test reading/writing with env unset", "[test_sparse_read_with_env_unset]") { unset_disable_file_locking(); unset_keep_write_file_handles_open(); write_array(); CHECK(consolidation_file_lock_exists()); // TILEDB_DISABLE_FILE_LOCK unset unset_disable_file_locking(); read_array(); // TILEDB_DISABLE_FILE_LOCK=1 set_disable_file_locking(); read_array(); } TEST_CASE_METHOD(SparseArrayEnvTestFixture, "Test reading/writing with env set", "[test_sparse_read_with_env_set]") { set_disable_file_locking(); set_keep_write_file_handles_open(); write_array(); CHECK(consolidation_file_lock_exists()); // TILEDB_DISABLE_FILE_LOCK unset unset_disable_file_locking(); read_array(); // TILEDB_DISABLE_FILE_LOCK=1 set_disable_file_locking(); read_array(); } class SparseArrayIntersectingTileTestFixture : SparseArrayTestFixture { public: // Write an array with intersecting tiles int write_array() { // Set array name set_array_name("sparse_test_intersecting_tile"); CHECK_RC(create_sparse_array_2D(4, 4, 0, 100, 0, 100, 3/*tile capacity*/, false, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR), TILEDB_OK); // Initialize the array TileDB_Array* tiledb_array; int rc = tiledb_array_init( tiledb_ctx_, &tiledb_array, array_name_.c_str(), TILEDB_ARRAY_WRITE, NULL, NULL, 0); if(rc != TILEDB_OK) return TILEDB_ERR; // Prepare buffers int buffer_a1[] = { 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 }; int64_t buffer_coords[] = { 23, 1, 23, 2, 23, 3, 23, 4, 30, 1, 30, 2, 30, 3, 30, 4, 45, 1, 45, 2, 45, 3, 45, 4, 97, 1, 97, 2, 97, 3, 97, 4 }; // Write to array const void* buffers[] = { buffer_a1, buffer_coords }; size_t buffer_sizes[2]; buffer_sizes[0] = sizeof(buffer_a1); buffer_sizes[1] = sizeof(buffer_coords); rc = tiledb_array_write(tiledb_array, buffers, buffer_sizes); if(rc != TILEDB_OK) return TILEDB_ERR; // Finalize the array rc = tiledb_array_finalize(tiledb_array); if(rc != TILEDB_OK) return TILEDB_ERR; return TILEDB_OK; } int read_array() { int *array = read_sparse_array_2D(23, 100, 0, 100, TILEDB_ARRAY_READ, 16); CHECK(array != NULL); for (int i=0; i<4; i++) { for (int j=0; j<4; j++) { CHECK(*(array+i*4+j) == j); } } delete [] array; array = read_sparse_array_2D(0, 100, 2, 2, TILEDB_ARRAY_READ, 4); CHECK(array != NULL); for (int i=0; i<4; i++) { CHECK(*(array+i) == 1); } delete [] array; return TILEDB_OK; } }; TEST_CASE_METHOD(SparseArrayIntersectingTileTestFixture, "Test case with intersecting tiles", "[test_sparse_array_intersecting_tiles]") { CHECK_RC(write_array(), TILEDB_OK); CHECK_RC(read_array(), TILEDB_OK); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/c_api/test_tiledb_utils.cc000066400000000000000000000322231453617025200252210ustar00rootroot00000000000000/** * @file test_tiledb_utils.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2019-2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Unit Tests for tiledb_utils.cc */ #include "catch.h" #include "tiledb.h" #include "tiledb_storage.h" #include "tiledb_utils.h" #include #include #include const std::string& workspace("WORKSPACE"); TEST_CASE_METHOD(TempDir, "Test initialize_workspace", "[initialize_workspace]") { std::string workspace_path = TileDBUtils::append_path(get_temp_dir(), workspace); TileDB_CTX *tiledb_ctx; REQUIRE(TileDBUtils::initialize_workspace(&tiledb_ctx, workspace_path, false) == 0); // OK std::string only_path = TileDBUtils::get_path(workspace_path); CHECK(!set_working_dir(tiledb_ctx, only_path)); std::string cwd = current_working_dir(tiledb_ctx);// path CHECK(cwd.size() > 0); CHECK(set_working_dir(tiledb_ctx, only_path + ".nonexistent")); CHECK(current_working_dir(tiledb_ctx) == cwd); CHECK(!tiledb_ctx_finalize(tiledb_ctx)); CHECK(TileDBUtils::workspace_exists(workspace_path)); CHECK(TileDBUtils::initialize_workspace(&tiledb_ctx, workspace_path, false) == 1); // EXISTS CHECK(!tiledb_ctx_finalize(tiledb_ctx)); CHECK(TileDBUtils::initialize_workspace(&tiledb_ctx, workspace_path, true) == 0); // OK CHECK(!create_file(tiledb_ctx, append(only_path,".new"), O_WRONLY | O_CREAT | O_SYNC, S_IRWXU)); CHECK(!tiledb_ctx_finalize(tiledb_ctx)); CHECK(TileDBUtils::initialize_workspace(&tiledb_ctx, append(workspace_path,".new"), true) == -1); // NOT_DIR CHECK(TileDBUtils::initialize_workspace(&tiledb_ctx, append(workspace_path,".new")) == -1); // NOT_DIR // Try paths with trailing slashes workspace_path = append(workspace_path,"1/"); only_path = TileDBUtils::get_path(workspace_path); CHECK(TileDBUtils::initialize_workspace(&tiledb_ctx, workspace_path) == 0); // OK CHECK(TileDBUtils::is_dir(workspace_path)); CHECK(TileDBUtils::is_file(append(workspace_path,TILEDB_WORKSPACE_FILENAME))); // Check out real dir auto real_workspace_path = TileDBUtils::real_dir(workspace_path); if (TileDBUtils::is_cloud_path(workspace_path)) { CHECK(real_workspace_path == workspace_path); } else { CHECK(real_workspace_path.find("//") == std::string::npos); CHECK(real_workspace_path.find("../") == std::string::npos); } auto non_existent_path = append(workspace_path,"2"); auto real_non_existent_path = TileDBUtils::real_dir(non_existent_path); if (TileDBUtils::is_cloud_path(workspace_path)) { CHECK(real_non_existent_path == non_existent_path); } else { CHECK(real_non_existent_path.find("//") == std::string::npos); CHECK(real_non_existent_path.find("../") == std::string::npos); } } TEST_CASE_METHOD(TempDir, "Test create_workspace", "[create_workspace]") { std::string workspace_path = TileDBUtils::append_path(get_temp_dir(), workspace); REQUIRE(!TileDBUtils::workspace_exists(workspace_path)); REQUIRE(TileDBUtils::get_dirs(workspace_path).size() == 0); REQUIRE(TileDBUtils::get_files(workspace_path).size() == 0); REQUIRE(TileDBUtils::create_workspace(workspace_path, false) == TILEDB_OK); CHECK(TileDBUtils::workspace_exists(workspace_path)); REQUIRE(TileDBUtils::get_dirs(workspace_path).size() == 0); REQUIRE(TileDBUtils::get_files(workspace_path).size() >= 1); CHECK(TileDBUtils::create_workspace(workspace_path, false) == 1); // EXISTS not REPLACED CHECK(TileDBUtils::workspace_exists(workspace_path)); CHECK(TileDBUtils::create_workspace(workspace_path, true) == TILEDB_OK); CHECK(TileDBUtils::workspace_exists(workspace_path)); std::string test_str("TESTING"); std::string test_file(TileDBUtils::append_path(workspace_path,"test")); CHECK(TileDBUtils::write_file(test_file, test_str.data(), 4) == TILEDB_OK); CHECK(TileDBUtils::is_file(test_file)); CHECK(TileDBUtils::create_workspace(workspace_path, true) == TILEDB_OK); CHECK(TileDBUtils::workspace_exists(workspace_path)); // test file should not exist as the existing workspace was overwritten CHECK(!TileDBUtils::is_file(test_file)); CHECK(!TileDBUtils::is_dir(test_file)); // Use defaults CHECK(TileDBUtils::create_workspace(workspace_path) == 1); // EXISTS not REPLACED CHECK(TileDBUtils::delete_dir(workspace_path) == TILEDB_OK); CHECK(TileDBUtils::create_workspace(workspace_path) == TILEDB_OK); // Try paths with trailing slashes workspace_path = append(workspace_path , "1/"); CHECK(TileDBUtils::create_workspace(workspace_path) == TILEDB_OK); CHECK(TileDBUtils::is_dir(workspace_path)); CHECK(TileDBUtils::is_file(append(workspace_path,TILEDB_WORKSPACE_FILENAME))); } TEST_CASE_METHOD(TempDir, "Test array exists", "[array_exists]") { std::string workspace_path = TileDBUtils::append_path(get_temp_dir(),workspace); std::string non_existent_array = std::string("non_existent_array"); // No workspace or array REQUIRE(!TileDBUtils::array_exists(workspace_path, non_existent_array)); REQUIRE(TileDBUtils::create_workspace(workspace_path, false) == TILEDB_OK); CHECK(TileDBUtils::workspace_exists(workspace_path)); // workspace exists but no array CHECK(!TileDBUtils::array_exists(workspace_path, non_existent_array)); std::string input_ws = std::string(TILEDB_TEST_DIR)+"/inputs/compatibility_gdb_pre100_ws"; std::string array_name("t0_1_2"); CHECK(TileDBUtils::workspace_exists(input_ws)); CHECK(TileDBUtils::array_exists(input_ws, array_name)); REQUIRE(TileDBUtils::get_dirs(input_ws).size() >= 1); REQUIRE(TileDBUtils::get_files(input_ws).size() >= 1); std::vector arrays = TileDBUtils::get_array_names(input_ws); CHECK(arrays.size()==1); CHECK(TileDBUtils::array_exists(input_ws, arrays[0])); arrays = TileDBUtils::get_array_names(input_ws+"/"); CHECK(arrays.size()==1); CHECK(TileDBUtils::array_exists(input_ws, arrays[0])); // Try paths with trailing slashes input_ws = input_ws + "/"; CHECK(TileDBUtils::workspace_exists(input_ws)); CHECK(TileDBUtils::array_exists(input_ws, array_name)); } TEST_CASE_METHOD(TempDir, "Test get fragment names", "[get_fragment_names]") { std::string workspace_path = TileDBUtils::append_path(get_temp_dir(),workspace); // No workspace or array or fragments REQUIRE(TileDBUtils::get_fragment_names(workspace_path).size() == 0); REQUIRE(TileDBUtils::create_workspace(workspace_path, false) == TILEDB_OK); CHECK(TileDBUtils::workspace_exists(workspace_path)); // No array or fragments CHECK(TileDBUtils::get_fragment_names(workspace_path).size() == 0); std::string input_ws = std::string(TILEDB_TEST_DIR)+"/inputs/compatibility_gdb_pre100_ws"; std::string array_name("t0_1_2"); CHECK(TileDBUtils::workspace_exists(input_ws)); CHECK(TileDBUtils::array_exists(input_ws, array_name)); // No fragments CHECK(TileDBUtils::get_fragment_names(input_ws).size() == 0); // TODO: Add input with one fragment } TEST_CASE_METHOD(TempDir, "Test multithreaded file utils", "[file_utils_multi_threads]") { std::string test_dir = TileDBUtils::append_path(get_temp_dir(),"test_dir"); CHECK(TileDBUtils::create_dir(test_dir) == TILEDB_OK); CHECK(!TileDBUtils::is_file(test_dir)); std::string test_file = TileDBUtils::append_path(test_dir,"test_file"); char buffer[1024]; memset(buffer, 'T', 1024); CHECK(TileDBUtils::write_file(test_file, buffer, 1024) == TILEDB_OK); CHECK(TileDBUtils::is_file(test_file)); CHECK(!TileDBUtils::is_dir(test_file)); // Define a lambda expression auto test_file_ops_fn = [](const std::string& dirname, const std::string& filename) { CHECK(TileDBUtils::is_dir(dirname)); CHECK(!TileDBUtils::is_file(dirname)); CHECK(TileDBUtils::is_file(filename)); CHECK(!TileDBUtils::is_dir(filename)); }; int num_threads = 16; std::vector threads; for (auto i=0; i filesize memset(buffer, 0, 1024); CHECK(TileDBUtils::read_file(filename, 1025, buffer, 256) == TILEDB_ERR); // read past filesize memset(buffer, 0, 1024); CHECK(TileDBUtils::read_file(filename, 256, buffer, 1024) == TILEDB_ERR); // read non-existent file CHECK(TileDBUtils::read_entire_file(filename+".1", &read_buffer, &length) == TILEDB_ERR); CHECK(TileDBUtils::read_file(filename+".1", 256, buffer, 256) == TILEDB_ERR); } TEST_CASE("Test create temp file", "[create_temp_file]") { char path[PATH_MAX]; CHECK(TileDBUtils::create_temp_filename(path, PATH_MAX) == TILEDB_OK); CHECK(TileDBUtils::delete_file(path) == TILEDB_OK); CHECK(!TileDBUtils::is_file(path)); } TEST_CASE_METHOD(TempDir, "Test move across filesystems", "[move_across_filesystems]") { std::string filename = TileDBUtils::append_path(get_temp_dir(),"test_file"); char buffer[1024]; memset(buffer, 'H', 1024); CHECK(TileDBUtils::write_file(filename, buffer, 1024) == TILEDB_OK); CHECK(TileDBUtils::is_file(filename)); char path[PATH_MAX]; CHECK(TileDBUtils::create_temp_filename(path, PATH_MAX) == TILEDB_OK); CHECK(TileDBUtils::move_across_filesystems(filename, path) == TILEDB_OK); CHECK(TileDBUtils::is_file(path)); CHECK(TileDBUtils::is_file(filename)); // src should not be deleted! CHECK(TileDBUtils::delete_file(path) == TILEDB_OK); CHECK(TileDBUtils::delete_file(filename) == TILEDB_OK); CHECK(!TileDBUtils::is_file(path)); CHECK(!TileDBUtils::is_file(filename)); } TEST_CASE("Test codec api", "[codec_api]") { void *handle; CHECK(TileDBUtils::create_codec(&handle, 999, 999) == TILEDB_ERR); // unsupported compression type CHECK(TileDBUtils::create_codec(&handle, TILEDB_GZIP, 0) == TILEDB_OK); CHECK(handle != nullptr); size_t size = 1024*1024*10; // 10M std::vector buffer(size); std::generate(buffer.begin(), buffer.end(), std::rand); void *compressed_buffer; size_t compressed_buffer_size; CHECK(TileDBUtils::compress(handle, &buffer[0], size, &compressed_buffer, compressed_buffer_size) == TILEDB_OK); std::vector decompressed_buffer(size); CHECK(TileDBUtils::decompress(handle, (unsigned char *)compressed_buffer, compressed_buffer_size, &decompressed_buffer[0], size) == TILEDB_OK); for (size_t i=0; i class TestCodecBasic : public Codec { public: using Codec::Codec; static Codec* createTestCodecBasic(const ArraySchema* array_schema, const int attribute_id, const bool is_offsets_compression) { return new TestCodecBasic(/*compression_level*/1); } int do_compress_tile(unsigned char* tile, size_t tile_size, void** tile_compressed, size_t& tile_compressed_size) { return TILEDB_CD_OK; } int do_decompress_tile(unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { return TILEDB_CD_OK; } }; TEST_CASE("Test codec static methods", "[codec_static]") { CHECK(Codec::get_default_level(TILEDB_GZIP) == TILEDB_COMPRESSION_LEVEL_GZIP); CHECK(Codec::get_default_level(INT_MAX) == -1); } TEST_CASE("Test codec basic", "[codec_basic]") { TestCodecBasic *codec_basic = new TestCodecBasic(0); void *dl_handle = codec_basic->get_dlopen_handle("non-existent-library"); CHECK(dl_handle == NULL); CHECK(!codec_basic->get_dlerror().empty()); dl_handle = codec_basic->get_dlopen_handle("non-existent-library", "5.2"); CHECK(dl_handle == NULL); CHECK(!codec_basic->get_dlerror().empty()); dl_handle = codec_basic->get_dlopen_handle("z"); CHECK(dl_handle); CHECK(codec_basic->get_dlerror().empty()); dl_handle = codec_basic->get_dlopen_handle("z", "1"); CHECK(dl_handle); CHECK(codec_basic->get_dlerror().empty()); dl_handle = codec_basic->get_dlopen_handle("z", "non-existent-version"); CHECK(dl_handle == NULL); CHECK(!codec_basic->get_dlerror().empty()); delete codec_basic; } TEST_CASE("Test zlib", "[codec-z]") { Codec* zlib = new CodecGzip(TILEDB_COMPRESSION_LEVEL_GZIP); unsigned char test_string[] = "HELLO"; unsigned char* buffer; size_t buffer_size; CHECK(zlib->compress_tile(test_string, 0, (void **)(&buffer), buffer_size) == TILEDB_CD_OK); CHECK(zlib->compress_tile(test_string, 6, (void **)(&buffer), buffer_size) == TILEDB_CD_OK); unsigned char* decompressed_string = (unsigned char*)malloc(6); CHECK(zlib->decompress_tile(buffer, buffer_size, decompressed_string, 6) == TILEDB_CD_OK); // Should get Z_BUF_ERROR CHECK(zlib->decompress_tile(buffer, buffer_size, decompressed_string, 4) == TILEDB_CD_ERR); CHECK(zlib->decompress_tile(buffer, 4, decompressed_string, 6) == TILEDB_CD_ERR); // Should get Z_DATA_ERROR CHECK(zlib->decompress_tile(test_string, 4, decompressed_string, 6) == TILEDB_CD_ERR); delete zlib; } #include "codec_lz4.h" TEST_CASE("Test lz4", "[codec-lz4]") { // Try lz4 with default compression Codec* lz4 = new CodecLZ4(TILEDB_COMPRESSION_LEVEL_LZ4); unsigned char test_string[] = "HELLO"; unsigned char* buffer; size_t buffer_size; CHECK(lz4->compress_tile(test_string, 0, (void **)(&buffer), buffer_size) == TILEDB_CD_OK); CHECK(lz4->compress_tile(test_string, 6, (void **)(&buffer), buffer_size) == TILEDB_CD_OK); unsigned char* decompressed_string = (unsigned char*)malloc(6); CHECK(lz4->decompress_tile(buffer, buffer_size, decompressed_string, 6) == TILEDB_CD_OK); CHECK(strlen((char *)decompressed_string) == 5); CHECK(strcmp((char *)decompressed_string, (char *)test_string) == 0); delete lz4; // Try lz4 with acceleration=2 lz4 = new CodecLZ4(2); CHECK(lz4->compress_tile(test_string, 0, (void **)(&buffer), buffer_size) == TILEDB_CD_OK); CHECK(lz4->compress_tile(test_string, 6, (void **)(&buffer), buffer_size) == TILEDB_CD_OK); memset(decompressed_string, 0, 6); CHECK(lz4->decompress_tile(buffer, buffer_size, decompressed_string, 6) == TILEDB_CD_OK); CHECK(strlen((char *)decompressed_string) == 5); CHECK(strcmp((char *)decompressed_string, (char *)test_string) == 0); free(decompressed_string); delete lz4; } TEST_CASE("Test plugin", "[codec-plugin]") { int compression_type = 15; CHECK(Codec::register_codec(compression_type, &TestCodecBasic::createTestCodecBasic) == TILEDB_CD_OK); CHECK(Codec::register_codec(compression_type, &TestCodecBasic::createTestCodecBasic) == TILEDB_CD_ERR); ArraySchema array_schema(NULL); const char* attributes[1] = {"this_attribute"}; array_schema.set_attributes(const_cast(attributes), 1); const int compression[2] = {compression_type, TILEDB_GZIP}; array_schema.set_compression(const_cast(compression)); CHECK(Codec::create(&array_schema, 0, false)); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/codec/test_codec_filter.cc000066400000000000000000000135641453617025200251710ustar00rootroot00000000000000/** * @file test_codec_filter.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2020 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Test pre and post compression filters */ #include "catch.h" #include "codec_filter_delta_encode.h" #include "codec_filter_bit_shuffle.h" #include "tiledb_constants.h" TEST_CASE("Test codec filter basic", "[codec_filter_basic]") { CodecFilter* codec_filter = new CodecFilter(TILEDB_INT32); CHECK(codec_filter->type() == TILEDB_INT32); CHECK(codec_filter->in_place()); CHECK(codec_filter->name() == ""); delete codec_filter; codec_filter = new CodecFilter(5, false); CHECK(codec_filter->type() == 5); CHECK(!codec_filter->in_place()); CHECK(codec_filter->name() == ""); size_t sz = 0; CHECK(codec_filter->code(NULL, 0, NULL, sz)); CHECK(codec_filter->code(NULL, 0)); CHECK(codec_filter->decode(NULL, 0, NULL, sz)); CHECK(codec_filter->decode(NULL, 0)); delete codec_filter; } class TestCodecFilterFixture { public: template size_t create_buffer(T** buffer, int stride, int length) { size_t sz = sizeof(T)*length; *buffer = reinterpret_cast(malloc(sz)); for (auto i=0l; i void check_buffer(T* buffer, int stride, int length, bool is_encode) { for (auto i=0l; i0) { CHECK(buffer[i*stride+j] == 1); } else { CHECK(buffer[i*stride+j] == (T)i); } } } } template inline bool instanceof(const T*) { return std::is_base_of::value; } template void test_filter(CodecFilter* codec_filter, T** buffer, int stride, int length) { size_t sz = create_buffer(buffer, stride, length); CHECK(sz == sizeof(T)*length); CHECK(codec_filter->code(reinterpret_cast(*buffer), sz) == 0); // Check output of encoding, only works with delta encoding for now if(instanceof(codec_filter)) { check_buffer(*buffer, stride, length, true); } CHECK(codec_filter->decode(reinterpret_cast(*buffer), sz) == 0); // Check output after decoding, it should be identical to the created buffer check_buffer(*buffer, stride, length, false); // code/decode should fail if length is not divisible by type_size and stride if ((sz-1) % sizeof(T) != 0) { CHECK(codec_filter->code(reinterpret_cast(*buffer), sz-1) != 0); CHECK(codec_filter->decode(reinterpret_cast(*buffer), sz-1) != 0); } if ((sz-1) % stride != 0) { CHECK(codec_filter->code(reinterpret_cast(*buffer), sz-1) != 0); CHECK(codec_filter->decode(reinterpret_cast(*buffer), sz-1) != 0); } } void test_filter(int filter_type, int type, int length, int stride=1) { CodecFilter *codec_filter = NULL; switch (filter_type) { case TILEDB_DELTA_ENCODE: codec_filter = new CodecDeltaEncode(type, stride); CHECK(codec_filter->name() == "Delta Encoding"); CHECK(codec_filter->in_place()); CHECK(codec_filter->type() == type); CHECK(reinterpret_cast(codec_filter)->stride() == stride); break; case TILEDB_BIT_SHUFFLE: codec_filter = new CodecBitShuffle(type); CHECK(!codec_filter->in_place()); CHECK(codec_filter->type() == type); break; default: return; } unsigned char* buffer = 0; switch (type) { case TILEDB_INT32: test_filter(codec_filter, reinterpret_cast(&buffer), stride, length); break; case TILEDB_INT64: test_filter(codec_filter, reinterpret_cast(&buffer), stride, length); break; case TILEDB_UINT32: test_filter(codec_filter, reinterpret_cast(&buffer), stride, length); break; case TILEDB_UINT64: test_filter(codec_filter, reinterpret_cast(&buffer), stride, length); break; } free(buffer); delete codec_filter; } TestCodecFilterFixture() {}; }; TEST_CASE_METHOD(TestCodecFilterFixture, "Test delta encoding", "[codec_filter_delta_encoding]") { test_filter(TILEDB_DELTA_ENCODE, TILEDB_INT64, 10, 1); test_filter(TILEDB_DELTA_ENCODE, TILEDB_INT32, 20, 2); test_filter(TILEDB_DELTA_ENCODE, TILEDB_UINT32, 9, 3); test_filter(TILEDB_DELTA_ENCODE, TILEDB_UINT64, 16, 4); test_filter(TILEDB_BIT_SHUFFLE, TILEDB_INT64, 10); test_filter(TILEDB_BIT_SHUFFLE, TILEDB_INT32, 20); test_filter(TILEDB_BIT_SHUFFLE, TILEDB_UINT32, 9, 3); test_filter(TILEDB_BIT_SHUFFLE, TILEDB_UINT64, 16); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/expressions/000077500000000000000000000000001453617025200224755ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/src/expressions/test_expressions.cc000066400000000000000000000545361453617025200264420ustar00rootroot00000000000000/** * @file test_expressions.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2019, 2022 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the Expression class */ #include "catch.h" #include "expression.h" #include "storage_posixfs.h" #include "tiledb.h" int rc; template class ArrayFixture { protected: const std::string ARRAYNAME = "test_sparse_array_1D"; const std::vector attribute_names = { "a1" }; const std::vector attribute_ids = { 0 }; const char* attr_names[1] = { "a1" }; int types[2] = { TILEDB_CHAR, TILEDB_INT32 }; int cell_val_nums[1] = { 1 }; T buffer_a1[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; void* buffers[1] { buffer_a1 }; size_t buffer_sizes[1] = { sizeof(buffer_a1) }; T expected_buffer[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; size_t buffer_var_a1_offsets[16] = { 0, 1*sizeof(T), 2*sizeof(T), 3*sizeof(T), 4*sizeof(T), 5*sizeof(T), 6*sizeof(T), 7*sizeof(T), 8*sizeof(T), 9*sizeof(T), 10*sizeof(T), 11*sizeof(T), 12*sizeof(T), 13*sizeof(T), 14*sizeof(T), 15*sizeof(T) }; T buffer_var_a1[16] = { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 }; void* var_buffers[2] = { buffer_var_a1_offsets, buffer_var_a1 }; size_t var_buffer_sizes[2] = { sizeof(buffer_var_a1_offsets), sizeof(buffer_var_a1) }; size_t expected_buffer_var_offsets[16] = { 0, 1*sizeof(T), 2*sizeof(T), 3*sizeof(T), 4*sizeof(T), 5*sizeof(T), 6*sizeof(T), 7*sizeof(T), 8*sizeof(T), 9*sizeof(T), 10*sizeof(T), 11*sizeof(T), 12*sizeof(T), 13*sizeof(T), 14*sizeof(T), 15*sizeof(T) }; T expected_buffer_var[16] = { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 }; size_t buffer_var_a1_offsets_1[8] = { 0, 1*sizeof(T), 3*sizeof(T), 5*sizeof(T), 8*sizeof(T), 11*sizeof(T), 12*sizeof(T), 13*sizeof(T) }; void* var_buffers_1[2] = { buffer_var_a1_offsets_1, buffer_var_a1 }; size_t var_buffer_sizes_1[2] = { sizeof(buffer_var_a1_offsets_1), sizeof(buffer_var_a1) }; PosixFS *posixfs_ = 0; ArraySchema *array_schema_ = 0; void setup(int cell_nums) { // Only cell_nums =0, 1 or TILEDB_VAR_NUM has been tested. But, this can // be extended to support all cell_nums bool supported_cell_nums = cell_nums == 1 || cell_nums == 2 || cell_nums == TILEDB_VAR_NUM; REQUIRE(supported_cell_nums); delete array_schema_; delete posixfs_; if (typeid(T)== typeid(char)) { types[0] = TILEDB_CHAR; } else if (typeid(T) == typeid(uint8_t)) { types[0] = TILEDB_UINT8; } else if (typeid(T) == typeid(uint16_t)) { types[0] = TILEDB_UINT16; } else if (typeid(T) == typeid(uint32_t)) { types[0] = TILEDB_UINT32; } else if (typeid(T) == typeid(uint64_t)) { types[0] = TILEDB_UINT64; } else if (typeid(T) == typeid(int8_t)) { types[0] = TILEDB_INT8; } else if (typeid(T) == typeid(int16_t)) { types[0] = TILEDB_INT16; } else if (typeid(T) == typeid(int) || typeid(T) == typeid(int32_t)) { types[0] = TILEDB_INT32; } else if (typeid(T) == typeid(int64_t)) { types[0] = TILEDB_INT64; } else if (typeid(T) == typeid(float)) { types[0] = TILEDB_FLOAT32; } else if (typeid(T) == typeid(double)) { types[0] = TILEDB_FLOAT64; } posixfs_ = new PosixFS(); array_schema_ = new ArraySchema(posixfs_); array_schema_->set_attributes(const_cast(attr_names), 1); cell_val_nums[0] = cell_nums; array_schema_->set_cell_val_num(cell_val_nums); array_schema_->set_types(types); array_schema_->set_dense(0); } ~ArrayFixture() { delete array_schema_; delete posixfs_; } void check_buffer(void **computed, size_t *computed_size, const T *expected, const size_t expected_size) { if (cell_val_nums[0] < TILEDB_VAR_NUM) { size_t buffer_size = *computed_size/sizeof(T); REQUIRE(buffer_size == expected_size); if (buffer_size) { T *buffer = reinterpret_cast(computed[0]); for (auto i=0u; i(computed[0]); for (auto i=0u; i(computed[1]); for (auto i=0u; i TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions Empty", "[expressions_empty]", char, int, float) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression(""); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::buffer_a1[0], 16); } SECTION("cell sizes = 2") { AF::setup(2); Expression expression(""); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::buffer_a1[0], 16); } SECTION("cell sizes = TILEDB_VAR_NUM") { AF::setup(TILEDB_VAR_NUM); Expression expression(""); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, &AF::expected_buffer_var[0], 16); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions All", "[expressions_all]", uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression("a1 >= 0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[0], 16); } SECTION("cell sizes = 2") { AF::setup(2); Expression expression("a1[0] >= 0 && a1[1] >=0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[0], 16); } SECTION("cell sizes = TILEDB_VAR_NUM") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] >= 0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, &AF::expected_buffer_var[0], 16); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions All", "[expressions_char]", char) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression("a1 >= 0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[0], 16); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions All - Out-of-bound expression", "[expressions_exception]", uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("cell sizes = 2") { AF::setup(2); Expression expression("a1[0] >= 0 && a1[2] >=0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_ERR); } SECTION("cell sizes = VAR") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] >= 0 && a1[2] >=0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_ERR); } } TEST_CASE_METHOD(ArrayFixture, "Test Expressions with ex", "[expression_exceptions]") { setup(1); Expression expression("a2 > a1"); // Attribute a2 does not exist CHECK_RC(expression.init(attribute_ids, array_schema_), TILEDB_ERR); Expression expression1("a1 === 22"); // Bad Operator CHECK_RC(expression1.init(attribute_ids, array_schema_), TILEDB_ERR); Expression expression2("a1 == 22"); // Evaluate without initialization REQUIRE(expression2.evaluate(buffers, buffer_sizes) == TILEDB_ERR); REQUIRE(expression2.evaluate_cell(buffers, buffer_sizes, NULL) == TILEDB_ERR); Expression expression3("a1 = 22"); // Assisgnment expression, not evaluating to a boolean CHECK_RC(expression3.init(attribute_ids, array_schema_), TILEDB_OK); REQUIRE(expression3.evaluate(buffers, buffer_sizes) == TILEDB_ERR); } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions With Dropped Cells from left", "[expressions_dropped_cells_left]", int, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression("a1 > 4"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[5], 11); } SECTION("cell sizes = 2") { AF::setup(1); Expression expression("a1[0] > 4"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[5], 11); } SECTION("cell sizes = TILEDB_VAR_NUM") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] > 20"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, &AF::expected_buffer_var[5], 11); } SECTION("cell sizes = TILEDB_VAR_NUM and selection expression includes a second value") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] > 20 && a1[0] > 21"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, &AF::expected_buffer_var[6], 10); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions With Dropped Cells from right", "[expressions_dropped_cells_right]", int, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression("a1 < 4"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[0], 4); } SECTION("cell sizes = 2") { AF::setup(2); Expression expression("a1[0] < 4"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, &AF::expected_buffer[0], 4); } SECTION("cell sizes = TILEDB_VAR_NUM") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] < 20"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, &AF::expected_buffer_var[0], 4); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions All Dropped Cells", "[expressions_all_dropped_cells]", int, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression("a1 > 16"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, 0, 0); } SECTION("cell sizes = 2") { AF::setup(2); Expression expression("a1[0] > 16"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::buffers, AF::buffer_sizes, 0, 0); } SECTION("cell sizes = TILEDB_VAR_NUM") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] > 32"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, 0, 0); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions Some Dropped Cells", "[expressions_some_cells]", int, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { SECTION("cell sizes = 1") { AF::setup(1); Expression expression("a1 == 15 || a1 == 7 || a1 == 0"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); const TestType expected_buffer[3] = { 0, 7, 15 }; AF::check_buffer(AF::buffers, AF::buffer_sizes, expected_buffer, 3); } SECTION("cell sizes = 2") { AF::setup(2); Expression expression("a1[0] == 6 || a1[1] == 11"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::buffers, AF::buffer_sizes) == TILEDB_OK); const TestType expected_buffer[4] = { 6, 7, 10, 11 }; AF::check_buffer(AF::buffers, AF::buffer_sizes, expected_buffer, 4); } SECTION("cell sizes = TILEDB_VAR_NUM") { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] == 26 || a1[0] == 29"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers, AF::var_buffer_sizes) == TILEDB_OK); const TestType expected_buffer[2] = { 26, 29 }; AF::check_buffer(AF::var_buffers, AF::var_buffer_sizes, expected_buffer, 2); } } TEMPLATE_TEST_CASE_METHOD(ArrayFixture, "Test Expressions with VAR_NUM and different offsets", "[expressions_different_offsets]", int, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, float, double) { AF::setup(TILEDB_VAR_NUM); Expression expression("a1[0] == 17 || a1[0] == 29"); REQUIRE(expression.init(AF::attribute_ids, AF::array_schema_) == TILEDB_OK); REQUIRE(expression.evaluate(AF::var_buffers_1, AF::var_buffer_sizes_1) == TILEDB_OK); CHECK(AF::var_buffer_sizes_1[0] == 2*sizeof(size_t)); CHECK(AF::var_buffer_sizes_1[1] == 5*sizeof(TestType)); // Check offset sizes first size_t expected_offsets[2] = {0, 2*sizeof(TestType)}; size_t *buffer_offsets = reinterpret_cast(AF::var_buffers_1[0]); for (auto i=0u; i<2; i++) { CHECK(buffer_offsets[i] == expected_offsets[i]); } // Check cell values TestType expected_values[5] = {17, 18, 29, 30, 31}; TestType *buffer_values = reinterpret_cast(AF::var_buffers_1[1]); for (auto i=0u; i<5; i++) { CHECK(buffer_values[i] == expected_values[i]); } } TEST_CASE("Test custom operator |= in Expression filters", "[custom_operators]") { const std::string array_name = "test_custom_operator_array"; const char *attr_names[] = { "a1" }; std::vector attribute_names = { "a1" }; std::vector attribute_ids = { 0 }; int types[2] = { TILEDB_CHAR, TILEDB_INT32 }; int cell_val_nums[1] = { TILEDB_VAR_NUM }; PosixFS posix_fs; ArraySchema array_schema(&posix_fs); array_schema.set_attributes(const_cast(attr_names), 1); array_schema.set_cell_val_num(cell_val_nums); array_schema.set_types(types); array_schema.set_dense(0); size_t buffer_offsets[] = { 0, 3, 6, 13 }; char buffer[17] = "A|CT|GA|C|T|GA|C"; // A|C T|G A|C|T|G A|C void *buffers[2] = { buffer_offsets, buffer }; size_t buffer_sizes[2] = { sizeof(buffer_offsets), sizeof(buffer) }; int64_t positions[] = { 0 }; Expression expression("a1 |= \"A\""); REQUIRE(expression.init(attribute_ids, &array_schema) == TILEDB_OK); REQUIRE(expression.evaluate_cell(buffers, buffer_sizes, positions)); // A|C positions[0] = 1; REQUIRE(!expression.evaluate_cell(buffers, buffer_sizes, positions)); // T|G positions[0] = 2; REQUIRE(expression.evaluate_cell(buffers, buffer_sizes, positions)); // A|C|T|G positions[0] = 3; REQUIRE(expression.evaluate_cell(buffers, buffer_sizes, positions)); // A|C } void check_evaluate_cell(const std::string& filter, ArraySchema *array_schema, std::vector& attribute_ids, void **buffers, size_t* buffer_sizes, std::vector positions, const std::vector& evaluations) { Expression expression(filter); REQUIRE(expression.init(attribute_ids, array_schema) == TILEDB_OK); for (auto i=0ul; i attribute_ids = { 0, 1, 2 }; int types[] = { TILEDB_INT32, TILEDB_CHAR, TILEDB_CHAR, TILEDB_INT64}; int cell_val_nums[] = { 3, TILEDB_VAR_NUM, TILEDB_VAR_NUM, 2 }; PosixFS posix_fs; ArraySchema array_schema(&posix_fs); array_schema.set_attributes(const_cast(attr_names), 3); array_schema.set_cell_val_num(cell_val_nums); array_schema.set_types(types); array_schema.set_dense(0); // GT int buffer_a1[] = { 0, 1, 1, 2, 1, 0, 0, 1, 2, 0, 0, 1, 1, 0, 1, 0, 0, 0, -1, 0, -1 }; // REF size_t buffer_a2_offsets[] = {0, 1, 2, 3, 8, 9, 10}; char buffer_a2[] = "TCGTAAAATG"; // T C G TAAAA T G // ALT size_t buffer_a3_offsets[] = { 0, 3, 7, 12, 15, 16, 17 }; char buffer_a3[] = "A|CTT|GA|C|TA|CAA"; // A|C TT|G A|C|T A|C A A void *buffers[] = { buffer_a1, buffer_a2_offsets, buffer_a2, buffer_a3_offsets, buffer_a3 }; size_t buffer_sizes[] = { sizeof(buffer_a1), sizeof(buffer_a2_offsets), sizeof(buffer_a2)-1, sizeof(buffer_a3_offsets), sizeof(buffer_a3)-1}; std::vector positions = { 0, 0, 0 }; // GT for positions 0 is "T|A" - het // GT for positions 1 is "G|C" - het // GT for positions 2 is "G|C" - het // GT for positions 3 is "TAAAA/A" - het // GT for positions 4 is "A/A" - homalt // GT for positions 5 is "G/G" - homref // GT for positions 6 is "./." - not homref or homalt or het SECTION("evaluation_with_empty") { Expression empty_expression("resolve(a1, a2, a3) &= \"\""); REQUIRE(empty_expression.init(attribute_ids, &array_schema) == TILEDB_OK); CHECK(empty_expression.evaluate_cell(buffers, buffer_sizes, positions.data()) == 0); //false Expression expression("resolve(a1) &= \"\""); REQUIRE(expression.init(attribute_ids, &array_schema) == TILEDB_ERR); } SECTION("evaluate") { check_evaluate_cell("ishet(a1)", EVAL_ARGS, {true, true, true, true, false, false, false}); check_evaluate_cell("ishomref(a1)", EVAL_ARGS, {false, false, false, false, false, true, false}); check_evaluate_cell("ishomalt(a1)", EVAL_ARGS, {false, false, false, false, true, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"G|C\"", EVAL_ARGS, {false, true, true, false, false, false, false}); check_evaluate_cell("ishet(a1) && resolve(a1, a2, a3) &= \"T|A\"", EVAL_ARGS, {true, false, false, false, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"T/A\"", EVAL_ARGS, {true, false, false, false, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"A/T\"", EVAL_ARGS, {true, false, false, false, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"TAAAA|A\"", EVAL_ARGS, {false, false, false, false, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"TAAAA/A\"", EVAL_ARGS, {false, false, false, true, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"A/TAAAA\"", EVAL_ARGS, {false, false, false, true, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"T\"", EVAL_ARGS, {true, false, false, false, false, false, false}); check_evaluate_cell("resolve(a1, a2, a3) &= \"./.\"", EVAL_ARGS, {false, false, false, false, false, false, true}); } } // TODO: Expressions not really supported as yet for dense arrays TEST_CASE_METHOD(ArrayFixture, "Test Expressions Dense", "[expressions_dense]") { setup(1); array_schema_->set_dense(1); Expression expression("a1==0 or a1 == 15"); REQUIRE(expression.init(attribute_ids, array_schema_) == TILEDB_ERR); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/expressions/test_filter_api.cc000066400000000000000000000417501453617025200261700ustar00rootroot00000000000000/** * @file test_filter_api.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2023 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for testing filter expressions from a GenomicsDB workspace */ #include "catch.h" #include "tiledb.h" std::string ws = std::string(TILEDB_TEST_DIR)+"/inputs/genomicsdb_ws"; std::string array = std::string(TILEDB_TEST_DIR)+"/inputs/genomicsdb_ws/1$1$249250621"; int num_attributes = 4; const char* attributes[] = { "REF", "ALT", "GT", TILEDB_COORDS }; // Sizes to be used for buffers size_t sizes[5] = { 1024, 40, 512, 4096, 4096 }; // filter expression that results in just one match for genomicsdb_ws std::string filters[5] = { "POS >= 0 && ROW >=0 && REF == \"G\" && GT[0]==1 && splitcompare(ALT, 124, \"T\") && resolve(GT, REF, ALT) &= \"T/T\"", "ISHOMALT && !ISHOMREF && REF == \"G\" && GT[0]==1 && splitcompare(ALT, 124, \"T\") && resolve(GT, REF, ALT) &= \"T/T\"", "!ISHET && !ISHOMALT == false && REF == \"G\" && splitcompare(ALT, 124, \"T\") && resolve(GT, REF, ALT) &= \"T/T\"", "ISHOMREF == false && ISHET == false && REF == \"G\" && ALT |= \"T\" && resolve(GT, REF, ALT) &= \"T\"", "POS==17384 && REF == \"G\" && ALT |= \"T\" && resolve(GT, REF, ALT) &= \"T/T\""}; // Only match expected for the following test cases char expected_REF = 'G'; char expected_ALT[3] = { 'T', '|', '&' }; int expected_GT_values[3] = { 1, 0, 1 }; int64_t expected_coords[2] = { 1, 17384 }; TEST_CASE("Test genomicsdb_ws filter with iterator api", "[genomicsdb_ws_filter_iterator]") { for (auto i=0; i<5; i++) { SECTION("Test filter expressions for iteration " + std::to_string(i)) { // Initialize tiledb context TileDB_CTX* tiledb_ctx; TileDB_Config tiledb_config; tiledb_config.home_ = ws.c_str(); CHECK_RC(tiledb_ctx_init(&tiledb_ctx, &tiledb_config), TILEDB_OK); size_t buffer_size = sizes[i]; size_t buffer_sizes[] = { buffer_size, buffer_size, buffer_size, buffer_size, buffer_size, buffer_size, buffer_size }; void *buffers[7]; auto nBuffers = sizeof(buffer_sizes)/sizeof(size_t); CHECK(nBuffers == 7); for (auto i=0u; i 0) { CHECK(coords_size == 16); CHECK(coords[0] == expected_coords[0]); CHECK(coords[1] == expected_coords[1]); } // Advance iterator CHECK_RC(tiledb_array_iterator_next(tiledb_array_it), TILEDB_OK); // Only one match, so iterator_end() should be true CHECK(tiledb_array_iterator_end(tiledb_array_it)); // Finalize the array CHECK_RC(tiledb_array_iterator_finalize(tiledb_array_it), TILEDB_OK); // Finalize context CHECK_RC(tiledb_ctx_finalize(tiledb_ctx), TILEDB_OK); // Deallocate memory for (auto i=0u; i0 && k filters = {"", // zlib 38s "ROW==10000", // 4s "POS==2084172232", // 4s "ROW==10000 && REF==\"T\"", // 5s "resolve(GT,REF,ALT)&=\"C|C\"", "resolve(GT,REF,ALT)&=\"C/C\"", "resolve(GT,REF,ALT)&=\"C\"", "ROW==10000 && REF==\"T\" && ALT|=\"C\"", // 6s "ROW==10000 && REF==\"T\" && ALT|=\"C\" && GT[0]==1", // 7s "ROW==10000 && REF==\"T\" && ALT|=\"C\" && resolve(GT,REF,ALT)&=\"C|C\"", // eval true 7s "ROW==10000 && REF==\"T\" && ALT|=\"C\" && resolve(GT,REF,ALT)&=\"C/C\"", // eval true 7s "ROW==10000 && REF==\"C\" && ALT|=\"T\" && resolve(GT,REF,ALT)&=\"C\"" // eval false 7s }; const int64_t subarray[] = {0ul, 200000ul, 2000000000ul, 2100000000ul}; for (auto filter_i=0u; filter_i void test_uri(const std::string path_uri_str, const std::string protocol, const std::string host, const int16_t nport, const std::string path, const std::unordered_map query) { uri path_uri(path_uri_str); INFO("URI=" << path_uri_str); CHECK_THAT(path_uri.protocol(), Equals(protocol)); CHECK_THAT(path_uri.host(), Equals(host)); CHECK(path_uri.nport() == nport); CHECK_THAT(path_uri.path(), Equals(path)); CHECK(path_uri.query().size() == query.size()); for(auto pair: query){ CHECK(path_uri.query().find(pair.first) != path_uri.query().end()); CHECK_THAT(pair.second, Equals(path_uri.query()[pair.first])); } } TEST_CASE("Test uri parsing", "[uri]") { REQUIRE_THROWS(new uri("")); REQUIRE_THROWS(new uri("gibberish")); REQUIRE_THROWS(new uri("foo://xxx:9999999/dfdfd")); std::unordered_map emptyMap; test_uri("hdfs://oda-master:9000/tmp", "hdfs", "oda-master", 9000, "/tmp", emptyMap); test_uri("hdfs://oda-master:9000/", "hdfs", "oda-master", 9000, "/", emptyMap); test_uri("hdfs://oda-master:9000", "hdfs", "oda-master", 9000, "", emptyMap); test_uri("hdfs://oda-master", "hdfs", "oda-master", 0, "", emptyMap); test_uri("hdfs://:9000", "hdfs", "", 9000, "", emptyMap); test_uri("hdfs://", "hdfs", "", 0, "", emptyMap); test_uri("hdfs:///", "hdfs", "", 0, "/", emptyMap); test_uri("hdfs:///tmp", "hdfs", "", 0, "/tmp", emptyMap); test_uri("s3://s3-bucket/tmp", "s3", "s3-bucket", 0, "/tmp", emptyMap); test_uri("gs://gcs-bucket/tmp", "gs", "gcs-bucket", 0, "/tmp", emptyMap); test_uri("fdfdfd://dfdfd/fdfdf?fdf=fdfdf", "fdfdfd", "dfdfd", 0, "/fdfdf", std::unordered_map{ {"fdf","fdfdf"}}); test_uri("hdfs://oda-master:9000/tmp?testQuery=val&anotherQuery=anotherval&lastQuery=lastval", "hdfs", "oda-master", 9000, "/tmp", std::unordered_map{ {"testQuery","val"},{"anotherQuery","anotherval"},{"lastQuery","lastval"}}); REQUIRE_THROWS(test_uri("hdfs://oda-master:9000/tmp?query&anotherquery=someval&otherquery&", "hdfs", "oda-master", 9000, "/tmp",emptyMap)); REQUIRE_THROWS(test_uri("fdfdfd://dfdfd/fdfdf?firstQ=firstval&secondQ", "fdfdfd", "dfdfd", 0, "/fdfdf",emptyMap)); REQUIRE_THROWS(test_uri("hdfs://oda-master:9000/tmp?query=someval&=otherquery", "hdfs", "oda-master", 9000, "/tmp",emptyMap)); REQUIRE_THROWS(test_uri("hdfs://oda-master:9000/tmp?initialQuery=val&&anotherval&latterQuery=lastval", "hdfs", "oda-master", 9000, "/tmp",emptyMap)); test_uri("hdfs://oda-master:9000/tmp?firstQuery=val&&secondquery=anotherval&thirdQuery=lastval", "hdfs", "oda-master", 9000, "/tmp", std::unordered_map{ {"firstQuery","val"},{"secondquery","anotherval"},{"thirdQuery","lastval"}}); test_uri("hdfs://oda-master:9000/tmp?firstQuery=val&&secondquery=anotherval&thirdQuery=last%20val", "hdfs", "oda-master", 9000, "/tmp", std::unordered_map{ {"firstQuery","val"},{"secondquery","anotherval"},{"thirdQuery","last val"}}); test_uri("hdfs://oda-master:9000/tmp?thirdQuery=someval123+otherval123==", "hdfs", "oda-master", 9000, "/tmp", std::unordered_map{ {"thirdQuery","someval123+otherval123=="}}); test_uri("hdfs://oda-master:9000/tmp?firstQuery=this%20is%20a%20field&secondquery=was%20it%20clear%20%28already%29%3F", "hdfs", "oda-master", 9000, "/tmp", std::unordered_map{ {"firstQuery","this is a field"},{"secondquery","was it clear (already)?"}}); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/misc/test_utils.cc000066400000000000000000000460561453617025200235670ustar00rootroot00000000000000/** * @file test_utils.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2016 MIT and Intel Corporation * @copyright Copyright (c) 2018-2019,2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the utility functionality. */ /* ****************************** */ /* TESTS */ /* ****************************** */ #include "catch.h" #include "storage_posixfs.h" #include "utils.h" #include #include #include /** Tests RLE compression (attribute). */ TEST_CASE("Test RLE compression(attribute)", "rle") { // Initializations unsigned char input[1000000] = {0}; size_t input_size = 0; unsigned char compressed[1000000]; size_t compressed_size = 0; unsigned char decompressed[1000000]; size_t decompressed_size = 0; size_t value_size; size_t run_size = 6; size_t compress_bound; int64_t rc; // === Attribute compression (value_size = sizeof(int)) === // value_size = sizeof(int); // Test empty bufffer rc = RLE_compress(input, input_size, compressed, compressed_size, value_size); CHECK(rc == 0); // Test input buffer invalid format input_size = 5; rc = RLE_compress(input, input_size, compressed, compressed_size, value_size); CHECK(rc == TILEDB_UT_ERR); // Test output buffer overflow input_size = 16; rc = RLE_compress(input, input_size, compressed, compressed_size, value_size); CHECK(rc == TILEDB_UT_ERR); // Test compress bound compress_bound = RLE_compress_bound(input_size, value_size); CHECK(compress_bound == input_size + ((input_size/value_size) * 2)); // Test all values unique (many unitary runs) for(int i=0; i<100; ++i) memcpy(input + i*value_size, &i, value_size); input_size = 100*value_size; compressed_size = RLE_compress_bound(input_size, value_size); rc = RLE_compress(input, input_size, compressed, compressed_size, value_size); REQUIRE(static_cast(rc) == compressed_size); decompressed_size = input_size; rc = RLE_decompress( compressed, compressed_size, decompressed, decompressed_size, value_size); REQUIRE(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test all values the same (a single long run) int v = 111; for(int i=0; i<100; ++i) memcpy(input + i*value_size, &v, value_size); input_size = 100*value_size; compressed_size = RLE_compress( input, input_size, compressed, compressed_size, value_size); REQUIRE(compressed_size == run_size); decompressed_size = input_size; rc = RLE_decompress( compressed, compressed_size, decompressed, decompressed_size, value_size); REQUIRE(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test a mix of short and long runs for(int i=0; i<10; ++i) memcpy(input + i*value_size, &i, value_size); for(int i=10; i<100; ++i) memcpy(input + i*value_size, &v, value_size); for(int i=100; i<110; ++i) memcpy(input + i*value_size, &i, value_size); input_size = 110*value_size; compressed_size = RLE_compress_bound(input_size, value_size); compressed_size = RLE_compress( input, input_size, compressed, compressed_size, value_size); REQUIRE(compressed_size == 21*run_size); decompressed_size = input_size; rc = RLE_decompress( compressed, compressed_size, decompressed, decompressed_size, value_size); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test when a run exceeds max run length for(int i=0; i<10; ++i) memcpy(input + i*value_size, &i, value_size); for(int i=10; i<70010; ++i) memcpy(input + i*value_size, &v, value_size); for(int i=70010; i<70030; ++i) memcpy(input + i*value_size, &i, value_size); input_size = 70030*value_size; compressed_size = RLE_compress_bound(input_size, value_size); compressed_size = RLE_compress( input, input_size, compressed, compressed_size, value_size); CHECK(compressed_size == 32*run_size); decompressed_size = input_size; rc = RLE_decompress( compressed, compressed_size, decompressed, decompressed_size, value_size); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // === Attribute compression (value_size = 2*sizeof(double)) === // value_size = 2*sizeof(double); run_size = value_size + 2; // Test a mix of short and long runs double j = 0.1, k = 0.2; for(int i=0; i<10; ++i) { j+= 10000.12; memcpy(input + 2*i*sizeof(double), &j, value_size); k+= 1000.12; memcpy(input + (2*i+1)*sizeof(double), &k, value_size); } j+= 10000.12; k+= 1000.12; for(int i=10; i<100; ++i) { memcpy(input + 2*i*sizeof(double), &j, value_size); memcpy(input + (2*i+1)*sizeof(double), &k, value_size); } for(int i=100; i<110; ++i) { j+= 10000.12; memcpy(input + 2*i*sizeof(double), &j, value_size); k+= 1000.12; memcpy(input + (2*i+1)*sizeof(double), &k, value_size); } input_size = 110*value_size; compressed_size = RLE_compress_bound(input_size, value_size); compressed_size = RLE_compress( input, input_size, compressed, compressed_size, value_size); REQUIRE(compressed_size == 21*run_size); decompressed_size = input_size; rc = RLE_decompress( compressed, compressed_size, decompressed, decompressed_size, value_size); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); } /** Tests RLE compression (coordinates, row-major cell order). */ TEST_CASE("Tests RLE compression (coordinates, row-major cell order)", "[test_RLE_coords_row]") { // Initializations unsigned char input[1000000] = {0}; unsigned char compressed[1000000]; unsigned char decompressed[1000000]; size_t input_size = 0; size_t compressed_size = 0; size_t decompressed_size = 0; size_t value_size; size_t coords_size; size_t run_size; size_t compress_bound; int dim_num = 2; int rc; // === Coordinates compression (row-major) === // value_size = sizeof(int); coords_size = dim_num*value_size; run_size = sizeof(int) + 2*sizeof(char); // Test empty bufffer rc = RLE_compress_coords_row( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(rc == 0); // Test input buffer invalid format input_size = 5; rc = RLE_compress_coords_row( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_ERR); // Test output buffer overflow input_size = 16; compressed_size = 0; rc = RLE_compress_coords_row( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_ERR); // Test compress bound input_size = 64; compress_bound = RLE_compress_bound_coords(input_size, value_size, dim_num); int64_t cell_num = input_size / coords_size; size_t compress_bound_expected = input_size + cell_num*(dim_num-1)*2 + sizeof(int64_t); CHECK(compress_bound == compress_bound_expected); // Test all values unique (many unitary runs) int v; for(int i=0; i<100; ++i) { v = i; memcpy(input + 2*i*value_size, &v, value_size); memcpy(input + (2*i+1)*value_size, &i, value_size); } input_size = 100*value_size*dim_num; compressed_size = RLE_compress_bound_coords(input_size, value_size, dim_num); rc = RLE_compress_coords_row( input, input_size, compressed, compressed_size, value_size, dim_num); REQUIRE(static_cast(rc) == compressed_size); decompressed_size = input_size; rc = RLE_decompress_coords_row( compressed, compressed_size, decompressed, decompressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test all values the same (a single long run) v = 111; for(int i=0; i<100; ++i) { memcpy(input + 2*i*value_size, &v, value_size); memcpy(input + (2*i+1)*value_size, &i, value_size); } input_size = 100*value_size*dim_num; compressed_size = RLE_compress_bound_coords(input_size, value_size, dim_num); compressed_size = RLE_compress_coords_row( input, input_size, compressed, compressed_size, value_size, dim_num); REQUIRE(compressed_size == 100*value_size + run_size + sizeof(int64_t)); decompressed_size = input_size; rc = RLE_decompress_coords_row( compressed, compressed_size, decompressed, decompressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test a mix of short and long runs for(int i=0; i<10; ++i) { v = i; memcpy(input + 2*i*value_size, &v, value_size); memcpy(input + (2*i+1)*value_size, &i, value_size); } v = 111; for(int i=10; i<90; ++i) { memcpy(input + 2*i*value_size, &v, value_size); memcpy(input + (2*i+1)*value_size, &i, value_size); } for(int i=90; i<100; ++i) { v = i; memcpy(input + 2*i*value_size, &v, value_size); memcpy(input + (2*i+1)*value_size, &i, value_size); } input_size = 100*value_size*dim_num; compressed_size = RLE_compress_bound_coords(input_size, value_size, dim_num); compressed_size = RLE_compress_coords_row( input, input_size, compressed, compressed_size, value_size, dim_num); REQUIRE(compressed_size == 100*value_size + 21*run_size + sizeof(int64_t)); decompressed_size = input_size; rc = RLE_decompress_coords_row( compressed, compressed_size, decompressed, decompressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); } /** Tests RLE compression (coordinates, column-major cell order). */ TEST_CASE("Tests RLE compression (coordinates, column-major cell order)", "[test_RLE_coords_col]") { // Initializations unsigned char input[1000000] = {0}; unsigned char compressed[1000000]; unsigned char decompressed[1000000]; size_t input_size = 0; size_t compressed_size = 0; size_t decompressed_size = 0; size_t value_size; size_t coords_size; size_t run_size; size_t compress_bound; int dim_num = 2; int rc; // === Coordinates compression (row-major) === // value_size = sizeof(int); coords_size = dim_num*value_size; run_size = sizeof(int) + 2*sizeof(char); // Test empty bufffer rc = RLE_compress_coords_col( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(rc == 0); // Test input buffer invalid format input_size = 5; rc = RLE_compress_coords_col( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_ERR); // Test output buffer overflow input_size = 16; compressed_size = 0; rc = RLE_compress_coords_col( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_ERR); // Test compress bound input_size = 64; compress_bound = RLE_compress_bound_coords(input_size, value_size, dim_num); int64_t cell_num = input_size / coords_size; size_t compress_bound_expected = input_size + cell_num*(dim_num-1)*2 + sizeof(int64_t); CHECK(compress_bound == compress_bound_expected); // Test all values unique (many unitary runs) int v; for(int i=0; i<100; ++i) { v = i; memcpy(input + 2*i*value_size, &i, value_size); memcpy(input + (2*i+1)*value_size, &v, value_size); } input_size = 100*value_size*dim_num; compressed_size = RLE_compress_bound_coords(input_size, value_size, dim_num); rc = RLE_compress_coords_col( input, input_size, compressed, compressed_size, value_size, dim_num); REQUIRE(static_cast(rc) == compressed_size); decompressed_size = input_size; rc = RLE_decompress_coords_col( compressed, compressed_size, decompressed, decompressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test all values the same (a single long run) v = 111; for(int i=0; i<100; ++i) { memcpy(input + 2*i*value_size, &i, value_size); memcpy(input + (2*i+1)*value_size, &v, value_size); } input_size = 100*value_size*dim_num; compressed_size = RLE_compress_bound_coords(input_size, value_size, dim_num); compressed_size = RLE_compress_coords_col( input, input_size, compressed, compressed_size, value_size, dim_num); CHECK(compressed_size == 100*value_size + run_size + sizeof(int64_t)); decompressed_size = input_size; rc = RLE_decompress_coords_col( compressed, compressed_size, decompressed, decompressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); // Test a mix of short and long runs for(int i=0; i<10; ++i) { v = i; memcpy(input + 2*i*value_size, &i, value_size); memcpy(input + (2*i+1)*value_size, &v, value_size); } v = 111; for(int i=10; i<90; ++i) { memcpy(input + 2*i*value_size, &i, value_size); memcpy(input + (2*i+1)*value_size, &v, value_size); } for(int i=90; i<100; ++i) { v = i; memcpy(input + 2*i*value_size, &i, value_size); memcpy(input + (2*i+1)*value_size, &v, value_size); } input_size = 100*value_size*dim_num; compressed_size = RLE_compress_bound_coords(input_size, value_size, dim_num); compressed_size = RLE_compress_coords_col( input, input_size, compressed, compressed_size, value_size, dim_num); REQUIRE(compressed_size == 100*value_size + 21*run_size + sizeof(int64_t)); decompressed_size = input_size; rc = RLE_decompress_coords_col( compressed, compressed_size, decompressed, decompressed_size, value_size, dim_num); CHECK(rc == TILEDB_UT_OK); CHECK_FALSE(memcmp(input, decompressed, input_size)); } TEST_CASE_METHOD(TempDir, "Test utils file system operations", "[test_utils_fs]") { PosixFS test_fs; PosixFS *fs = &test_fs; std::string temp_dir = get_temp_dir()+"/temp"; CHECK(create_dir(fs, "/non-existent-dir/dir") == TILEDB_UT_ERR); CHECK(create_file(fs, "/non-existent-dir/file", 0, 0) == TILEDB_UT_ERR); CHECK(write_to_file(fs, "/non-existent-dir/file", "Hello", 6) == TILEDB_UT_ERR); char check_str[6]; CHECK(read_from_file(fs, "/non-existent-dir/file", 0, &check_str[0], 6) == TILEDB_UT_ERR); CHECK(!sync_path(fs, "/non-existent-dir/file")); // This is OK for an non-existent path CHECK(delete_file(fs, "/non-existent-dir/file") == TILEDB_UT_ERR); CHECK(delete_dir(fs, "/non-existent-dir/dir") == TILEDB_UT_ERR); CHECK(move_path(fs, "/non-existent-dir/old", "/non-existent-dir/new") == TILEDB_UT_ERR); CHECK(!create_dir(fs, temp_dir)); CHECK(is_dir(fs, temp_dir)); const std::string path = temp_dir+"/foo"; CHECK(!create_file(fs, path, O_WRONLY|O_CREAT, S_IRWXU)); CHECK(is_file(fs, path)); CHECK(!write_to_file(fs, path, "Hello", 6)); CHECK(!sync_path(fs, path)); CHECK(!read_from_file(fs, path, 0, &check_str[0], 6)); CHECK(file_size(fs, path) == 6); CHECK(strcmp(&check_str[0], "Hello") == 0); const std::string new_path = temp_dir+"/new"; CHECK(!move_path(fs, path, new_path)); CHECK(is_file(fs, new_path)); CHECK(file_size(fs, new_path) == 6); CHECK(!delete_file(fs, new_path)); } TEST_CASE("Test empty value concept", "[empty_cell_val]") { char char_max = get_tiledb_empty_value(); CHECK(char_max == CHAR_MAX); int8_t int8_max = get_tiledb_empty_value(); CHECK(int8_max == INT8_MAX); int16_t int16_max = get_tiledb_empty_value(); CHECK(int16_max == INT16_MAX); int32_t int32_max = get_tiledb_empty_value(); CHECK(int32_max == INT32_MAX); int64_t int64_max = get_tiledb_empty_value(); CHECK(int64_max == INT64_MAX); uint64_t uint8_max = get_tiledb_empty_value(); CHECK(uint8_max == UINT8_MAX); uint16_t uint16_max = get_tiledb_empty_value(); CHECK(uint16_max == UINT16_MAX); uint32_t uint32_max = get_tiledb_empty_value(); CHECK(uint32_max == UINT32_MAX); uint64_t uint64_max = get_tiledb_empty_value(); CHECK(uint64_max == UINT64_MAX); float float_max = get_tiledb_empty_value(); CHECK(float_max == FLT_MAX); double double_max = get_tiledb_empty_value(); CHECK(double_max == DBL_MAX); } TEST_CASE("Test storage URIs", "[storage_uris]") { CHECK(!is_supported_cloud_path("gibberish://ddd/d")); CHECK(is_supported_cloud_path("hdfs://ddd/d")); CHECK(is_supported_cloud_path("s3://ddd/d")); CHECK(is_supported_cloud_path("gs://ddd/d")); CHECK(is_supported_cloud_path("wasb://ddd/d")); CHECK(is_supported_cloud_path("wasbs://ddd/d")); CHECK(is_supported_cloud_path("abfs://ddd/d")); CHECK(is_supported_cloud_path("abfss://ddd/d")); CHECK(is_supported_cloud_path("adl://ddd/d")); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/storage_manager/000077500000000000000000000000001453617025200232515ustar00rootroot00000000000000genomicsdb-0.0~git20231212.9d7ddd0/test/src/storage_manager/test_azure_blob_storage.cc000066400000000000000000000305451453617025200304760ustar00rootroot00000000000000/** * @file test_azure_blob_storage.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * @copyright Copyright (c) 2023 dātma, inc™ * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the AzureBlob class */ #include "catch.h" #include "storage_azure_blob.h" #include "uri.h" #include "utils.h" #include #include #include #include #include class AzureBlobTestFixture { protected: TempDir *temp_dir = NULL; AzureBlob *azure_blob = NULL; AzureBlobTestFixture() { if (is_azure_blob_storage_path(get_test_dir())) { try { temp_dir = new TempDir(); azure_blob = new AzureBlob(TileDBUtils::append_path(temp_dir->get_temp_dir(), "test_azure_blob")); CHECK(!azure_blob->locking_support()); } catch(...) { INFO("Azure Blob Storage could not be credentialed. Set env AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY"); } } INFO("Azure Blob Storage not specified as a --test-dir option"); } ~AzureBlobTestFixture() { if (azure_blob) { delete azure_blob; } if (temp_dir) { delete temp_dir; } } }; TEST_CASE("Test AzureBlob constructor", "[constr]") { CHECK_THROWS(new AzureBlob("wasbs://my_container/path")); CHECK_THROWS(new AzureBlob("az://my_container@my_account.blob.core.windows.net/path")); CHECK_THROWS(new AzureBlob("az://my_container@blob.core.windows.net/path")); // CHECK_THROWS(new AzureBlob("az://non-existent-container@blob.core.windows.met/path")); if (getenv("AZURE_STORAGE_ACCOUNT")) { unsetenv( "AZURE_STORAGE_ACCOUNT"); } std::string sas_token = "AZURE_STORAGE_SAS_TOKEN=non-existent-token"; CHECK(putenv(const_cast(sas_token.c_str())) == 0); // CHECK_THROWS(new AzureBlob("az://my_container@my_account.blob.core.windows.net/path")); } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob cwd", "[cwd]") { if (azure_blob == nullptr) { return; } REQUIRE(azure_blob->current_dir().length() > 0); REQUIRE(azure_blob->create_dir(azure_blob->current_dir()) == TILEDB_FS_OK); // create_dir is a no-op for AzureBlob, so is_dir will return false CHECK(!azure_blob->is_dir(azure_blob->current_dir())); REQUIRE(!azure_blob->is_file(azure_blob->current_dir())); REQUIRE(azure_blob->real_dir(azure_blob->current_dir()).length() > 0); } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob real_dir", "[real-dir]") { if (azure_blob == nullptr) { return; } CHECK(azure_blob->real_dir("").compare(azure_blob->current_dir()) == 0); CHECK(azure_blob->real_dir("xxx").compare(azure_blob->current_dir()+"/xxx") == 0); CHECK(azure_blob->real_dir("xxx/yyy").compare(azure_blob->current_dir()+"/xxx/yyy") == 0); CHECK(azure_blob->real_dir("/xxx/yyy").compare("xxx/yyy") == 0); azure_uri test_uri(get_test_dir()); CHECK(azure_blob->real_dir(get_test_dir()).compare(test_uri.path().substr(1)) == 0); CHECK_THROWS(azure_blob->real_dir("xxx://yyy")); } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob dir", "[dir]") { if (azure_blob == nullptr) { return; } std::string test_dir("dir"); CHECK_RC(azure_blob->create_dir(test_dir), TILEDB_FS_OK); // create_dir is a no-op for AzureBlob, so is_dir will return false CHECK(!azure_blob->is_dir(test_dir)); CHECK(!azure_blob->is_file(test_dir)); // Files can be created without parent dir existence on AzureBlob CHECK_RC(azure_blob->create_file(test_dir+"/foo", 0, 0), TILEDB_FS_OK); CHECK(azure_blob->is_dir(test_dir)); CHECK(azure_blob->file_size(test_dir) == TILEDB_FS_ERR); CHECK(azure_blob->get_dirs(test_dir).size() == 0); CHECK(azure_blob->get_dirs("non-existent-dir").size() == 0); // TBD: move_path std::string new_dir = test_dir+"-new"; CHECK_THROWS(azure_blob->move_path(test_dir, new_dir)); CHECK_RC(azure_blob->sync_path(test_dir), TILEDB_FS_OK); // No support for returning errors for non-existent paths CHECK_RC(azure_blob->sync_path("non-existent-dir"), TILEDB_FS_OK); CHECK_RC(azure_blob->delete_dir(test_dir), TILEDB_FS_OK); // No support for returning errors for non-existent paths CHECK_RC(azure_blob->delete_dir("non-existent-dir"), TILEDB_FS_OK); CHECK(!azure_blob->is_dir(test_dir)); CHECK(!azure_blob->is_file(test_dir)); } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob file", "[file]") { if (azure_blob == nullptr) { return; } std::string test_dir("file"); CHECK_RC(azure_blob->create_dir(test_dir), 0); CHECK_RC(azure_blob->create_file(test_dir+"/foo", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(azure_blob->is_file(test_dir+"/foo")); CHECK(!azure_blob->is_dir(test_dir+"/foo")); // Cannot create_dir if file already exists CHECK(azure_blob->create_dir(test_dir+"/foo") == TILEDB_FS_ERR); CHECK(azure_blob->file_size(test_dir+"/foo") == 0); CHECK(azure_blob->file_size(test_dir+"/foo1") == TILEDB_FS_ERR); CHECK(azure_blob->get_files(test_dir).size() == 1); CHECK(azure_blob->get_files("non-existent-dir").size() == 0); CHECK_RC(azure_blob->create_file(test_dir+"/foo1", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(azure_blob->get_files(test_dir).size() == 2); CHECK_RC(azure_blob->sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(azure_blob->sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(azure_blob->delete_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(azure_blob->delete_file(test_dir+"/foo1"), TILEDB_FS_OK); CHECK_RC(azure_blob->delete_file(test_dir+"/foo2"), TILEDB_FS_ERR); CHECK(!azure_blob->is_file(test_dir+"/foo1")); CHECK(!azure_blob->is_file(test_dir+"/foo2")); CHECK(!azure_blob->is_dir(test_dir+"/foo1")); CHECK(!azure_blob->is_dir(test_dir+"/foo2")); } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob read/write file", "[read-write]") { if (azure_blob == nullptr) { return; } std::string test_dir("read_write"); CHECK_RC(azure_blob->create_dir(test_dir), TILEDB_FS_OK); CHECK_RC(azure_blob->write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); CHECK_RC(azure_blob->close_file(test_dir+"/foo"), TILEDB_FS_OK); REQUIRE(azure_blob->is_file(test_dir+"/foo")); CHECK(azure_blob->file_size(test_dir+"/foo") == 5); void *buffer = malloc(20); memset(buffer, 'X', 20); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer, 0), TILEDB_FS_OK); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer, 2), TILEDB_FS_OK); CHECK(((char *)buffer)[0] == 'h'); CHECK(((char *)buffer)[1] == 'e'); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer, 5), TILEDB_FS_OK); CHECK(((char *)buffer)[4] == 'o'); // Reading past filesize does not seem to affect download_blob_to_stream/buffer. It // returns successfully even though Posix/HDFS/S3 data stores behave differently. We // could make the behavior identical on the stores, but for now leaving it to the clients // to not read past the file size. CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer, 6), TILEDB_FS_OK); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 3, buffer, 2), TILEDB_FS_OK); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 3, buffer, 6), TILEDB_FS_OK); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 6, buffer, 2), TILEDB_FS_ERR); CHECK_RC(azure_blob->write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); CHECK_RC(azure_blob->write_to_file(test_dir+"/foo", " there ", 6), TILEDB_FS_OK); CHECK_RC(azure_blob->close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK(azure_blob->file_size(test_dir+"/foo") == 11); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer, 11), TILEDB_FS_OK); CHECK(((char *)buffer)[10] == 'e'); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer, 11), TILEDB_FS_OK); CHECK(((char *)buffer)[10] == 'e'); CHECK_RC(azure_blob->close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(azure_blob->delete_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(azure_blob->sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(azure_blob->read_from_file(test_dir+"/non-existent-file", 0, buffer, 5), TILEDB_FS_ERR); CHECK_RC(azure_blob->read_from_file("non-existent-dir/foo", 0, buffer, 5), TILEDB_FS_ERR); // AzureBlob can write to non-existent dirs - create_dir really is a no-op CHECK_RC(azure_blob->write_to_file("non-existent-dir/foo", "hello", 5), TILEDB_FS_OK); CHECK_RC(azure_blob->close_file("non-existent-dir/foo"), TILEDB_FS_OK); free(buffer); } /* Test fails on Centos7 with TILEDB_MAX_STREAM_SIZE=32, but succeeds on Ubuntu and MacOS! */ TEST_CASE_METHOD(AzureBlobTestFixture, "Test performance of AzureBlob reads of small files", "[.][!mayfail][read-write-small]") { if (azure_blob == nullptr) { return; } uint num_iterations = 512; // May have to increase iterations to reproduce failure on Centos 7, can go to num_iterations=2000! std::string test_dir("read_write_small"); std::vector buffer(100, 22); for (auto i=0L; iwrite_to_file(filename, buffer.data(), buffer.size()*sizeof(int)) == TILEDB_FS_OK); REQUIRE(azure_blob->close_file(filename) == TILEDB_FS_OK); REQUIRE(azure_blob->read_from_file(test_dir+"/"+std::to_string(i), 0, buffer.data(), buffer.size()*sizeof(int)) == TILEDB_FS_OK); } } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob large read/write file", "[read-write-large]") { if (azure_blob == nullptr) { return; } std::string test_dir("read_write_large"); // size_t size = ((size_t)TILEDB_UT_MAX_WRITE_COUNT)*4 size_t size = ((size_t)TILEDB_UT_MAX_WRITE_COUNT); void *buffer = malloc(size); if (buffer) { memset(buffer, 'B', size); REQUIRE(azure_blob->create_dir(test_dir) == TILEDB_FS_OK); CHECK_RC(azure_blob->write_to_file(test_dir+"/foo", buffer, size), TILEDB_FS_OK); CHECK_RC(azure_blob->close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK(azure_blob->is_file(test_dir+"/foo")); CHECK((size_t)azure_blob->file_size(test_dir+"/foo") == size); void *buffer1 = malloc(size); if (buffer1) { memset(buffer1, 0, size); CHECK_RC(azure_blob->read_from_file(test_dir+"/foo", 0, buffer1, size), TILEDB_FS_OK); CHECK(memcmp(buffer, buffer1, size) == 0); free(buffer1); } free(buffer); } } TEST_CASE_METHOD(AzureBlobTestFixture, "Test AzureBlob parallel operations", "[parallel]") { if (azure_blob == nullptr) { return; } std::string test_dir("parallel"); REQUIRE(azure_blob->create_dir(test_dir) == TILEDB_FS_OK); bool complete = true; uint iterations = 2; size_t size = 10*1024*1024; #pragma omp parallel for for (uint i=0; iwrite_to_file(filename, buffer, size), TILEDB_FS_OK); free(buffer); } else { complete = false; } } } CHECK_RC(azure_blob->sync_path(test_dir), TILEDB_FS_OK); if (complete) { #pragma omp parallel for for (uint i=0; iclose_file(filename), TILEDB_FS_OK); CHECK(azure_blob->is_file(filename)); CHECK((size_t)azure_blob->file_size(filename) == size*2); } } CHECK_RC(azure_blob->delete_dir(test_dir), TILEDB_FS_OK); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/storage_manager/test_gcs_storage.cc000066400000000000000000000270651453617025200271310ustar00rootroot00000000000000/** * @file test_gcs_storage.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the GCS storage class */ #include "catch.h" #include "storage_gcs.h" #include "uri.h" #include "utils.h" #include #include #include #include class GCSTestFixture { protected: TempDir *temp_dir = NULL; GCS *gcs_instance = NULL; GCSTestFixture() { if (is_gcs_path(get_test_dir())) { try { temp_dir = new TempDir(); std::string home_dir = temp_dir->get_temp_dir()+"/test_gcs"; gcs_instance = new GCS(home_dir); CHECK(!gcs_instance->locking_support()); } catch(...) { INFO("GCS Storage could not be credentialed"); } } INFO("GCS Storage not specified as a --test-dir option"); } ~GCSTestFixture() { if (gcs_instance) { delete gcs_instance; } if (temp_dir) { delete temp_dir; } } }; TEST_CASE("Test GCS constructor", "[constr]") { CHECK_THROWS(new GCS("gcs://my_container/path")); CHECK_THROWS(new GCS("az://my_container@my_account.blob.core.windows.net/path")); CHECK_THROWS(new GCS("gs://my_container/path")); } TEST_CASE_METHOD(GCSTestFixture, "Test GCS cwd", "[cwd]") { if (gcs_instance == nullptr) { return; } REQUIRE(gcs_instance->current_dir().length() > 0); REQUIRE(gcs_instance->create_dir(gcs_instance->current_dir()) == TILEDB_FS_OK); REQUIRE(gcs_instance->is_dir(gcs_instance->current_dir())); REQUIRE(!gcs_instance->is_file(gcs_instance->current_dir())); REQUIRE(gcs_instance->real_dir(gcs_instance->current_dir()).length() > 0); } TEST_CASE_METHOD(GCSTestFixture, "Test GCS real_dir", "[real_dir]") { if (gcs_instance == nullptr) { return; } CHECK(gcs_instance->real_dir("").compare(gcs_instance->current_dir()) == 0); CHECK(gcs_instance->real_dir("xxx").compare(gcs_instance->current_dir()+"/xxx") == 0); CHECK(gcs_instance->real_dir("xxx/yyy").compare(gcs_instance->current_dir()+"/xxx/yyy") == 0); CHECK(gcs_instance->real_dir("/xxx/yyy").compare("xxx/yyy") == 0); gcs_uri test_uri(get_test_dir()); CHECK(gcs_instance->real_dir(get_test_dir()).compare(test_uri.path().substr(1)) == 0); CHECK_THROWS(gcs_instance->real_dir("xxx://yyy")); } TEST_CASE_METHOD(GCSTestFixture, "Test GCS dir", "[dir]") { if (gcs_instance == nullptr) { return; } CHECK(!gcs_instance->is_dir("non-existent-dir")); CHECK(!gcs_instance->is_dir("non-existent-parent-dir/dir")); std::string test_dir("dir"); CHECK_RC(gcs_instance->create_dir(test_dir), TILEDB_FS_OK); CHECK(gcs_instance->is_dir(test_dir)); CHECK(!gcs_instance->is_file(test_dir)); CHECK_RC(gcs_instance->create_dir(test_dir), TILEDB_FS_ERR); // Dir already exists CHECK_RC(gcs_instance->create_file(test_dir, 0, 0), TILEDB_FS_ERR); CHECK_RC(gcs_instance->create_file(gcs_instance->real_dir(test_dir), 0, 0), TILEDB_FS_ERR); CHECK(gcs_instance->file_size(test_dir) == TILEDB_FS_ERR); CHECK(gcs_instance->get_dirs(test_dir).size() == 0); CHECK(gcs_instance->get_files(test_dir).size() == 0); CHECK(gcs_instance->get_dirs("non-existent-dir").size() == 0); CHECK(gcs_instance->get_files("non-existent-dir").size() == 0); // Try directories with trailing slash std::string test_dir1 = test_dir + "1/"; CHECK_RC(gcs_instance->create_dir(test_dir1), TILEDB_FS_OK); CHECK(gcs_instance->is_dir(test_dir1)); CHECK(gcs_instance->is_dir(test_dir + "1")); CHECK(!gcs_instance->is_file(test_dir1)); CHECK(!gcs_instance->is_file(test_dir1 + "1")); // TBD: move_path std::string new_dir = test_dir+"-new"; CHECK_THROWS(gcs_instance->move_path(test_dir, new_dir)); CHECK_RC(gcs_instance->sync_path(test_dir), TILEDB_FS_OK); // No support for returning errors for non-existent paths CHECK_RC(gcs_instance->sync_path("non-existent-dir"), TILEDB_FS_OK); CHECK_RC(gcs_instance->delete_dir(test_dir), TILEDB_FS_OK); CHECK_RC(gcs_instance->delete_dir("non-existent-dir"), TILEDB_FS_ERR); CHECK(!gcs_instance->is_dir(test_dir)); CHECK(!gcs_instance->is_file(test_dir)); } TEST_CASE_METHOD(GCSTestFixture, "Test GCS file", "[file]") { if (gcs_instance == nullptr) { return; } std::string test_dir("file"); CHECK_RC(gcs_instance->create_dir(test_dir), 0); REQUIRE(gcs_instance->is_dir(test_dir)); CHECK_RC(gcs_instance->create_file(test_dir+"/foo", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(gcs_instance->is_file(test_dir+"/foo")); CHECK(!gcs_instance->is_dir(test_dir+"/foo")); CHECK(gcs_instance->file_size(test_dir+"/foo") == 0); CHECK(gcs_instance->file_size(test_dir+"/foo1") == TILEDB_FS_ERR); CHECK(gcs_instance->get_files(test_dir).size() == 1); CHECK(gcs_instance->get_files("non-existent-dir").size() == 0); CHECK_RC(gcs_instance->create_file(test_dir+"/foo1", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK_RC(gcs_instance->create_dir(test_dir+"/subdir"), TILEDB_FS_OK); CHECK_RC(gcs_instance->create_file(test_dir+"/subdir/subfoo", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(gcs_instance->get_files(test_dir).size() == 2); CHECK_RC(gcs_instance->sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(gcs_instance->sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(gcs_instance->delete_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(gcs_instance->delete_file(test_dir+"/foo1"), TILEDB_FS_OK); CHECK_RC(gcs_instance->delete_file(test_dir+"/foo2"), TILEDB_FS_ERR); CHECK(!gcs_instance->is_file(test_dir+"/foo1")); CHECK(!gcs_instance->is_file(test_dir+"/foo2")); CHECK(!gcs_instance->is_dir(test_dir+"/foo1")); CHECK(!gcs_instance->is_dir(test_dir+"/foo2")); } TEST_CASE_METHOD(GCSTestFixture, "Test GCS read/write file", "[read-write]") { if (gcs_instance == nullptr) { return; } std::string test_dir("read_write"); CHECK_RC(gcs_instance->create_dir(test_dir), TILEDB_FS_OK); REQUIRE(gcs_instance->is_dir(test_dir)); CHECK_RC(gcs_instance->write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); CHECK_RC(gcs_instance->sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(gcs_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); auto files = gcs_instance->get_files(test_dir); for(auto f: files) { std::cerr << "file=" << f << std::endl; } REQUIRE(gcs_instance->is_file(test_dir+"/foo")); CHECK(gcs_instance->file_size(test_dir+"/foo") == 5); void *buffer = malloc(20); memset(buffer, 'X', 20); CHECK_RC(gcs_instance->read_from_file(test_dir+"/foo", 0, buffer, 0), TILEDB_FS_OK); CHECK_RC(gcs_instance->read_from_file(test_dir+"/foo", 0, buffer, 2), TILEDB_FS_OK); CHECK(((char *)buffer)[0] == 'h'); CHECK(((char *)buffer)[1] == 'e'); CHECK_RC(gcs_instance->read_from_file(test_dir+"/foo", 0, buffer, 5), TILEDB_FS_OK); CHECK(((char *)buffer)[4] == 'o'); CHECK_RC(gcs_instance->read_from_file(test_dir+"/foo", 0, buffer, 6), TILEDB_FS_ERR); CHECK_RC(gcs_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); memset(buffer, 'X', 20); CHECK_RC(gcs_instance->write_to_file(test_dir+"/foo1", buffer, 20), TILEDB_FS_OK); CHECK_RC(gcs_instance->close_file(test_dir+"/foo1"), TILEDB_FS_OK); REQUIRE(gcs_instance->is_file(test_dir+"/foo1")); for (int i=0; i<4; i++) { void *buf = malloc(5); memset(buf, 'Z', 5); CHECK_RC(gcs_instance->read_from_file(test_dir+"/foo1", i*5, buf, 5), TILEDB_FS_OK); for (int j=0; j<5; j++) { CHECK(((char *)buf)[j] == 'X'); } free(buf); } // This part is different from Azure Blob Storage where you are allowed to write as many chunks in any size // No overwriting allowed with GCS, so only write-once semantics allowed after a file is committed CHECK_RC(gcs_instance->write_to_file(test_dir+"/foo2", "hello", 5), TILEDB_FS_OK); CHECK_RC(gcs_instance->write_to_file(test_dir+"/foo2", " there ", 6), TILEDB_FS_ERR); // Prev part should at least be 5M, otherwise should be aborted CHECK_RC(gcs_instance->close_file(test_dir+"/foo2"), TILEDB_FS_OK); CHECK(gcs_instance->file_size(test_dir+"/foo2") == 5); // Previous file size as the current set of writes failed CHECK_RC(gcs_instance->read_from_file(test_dir+"/non-existent-file", 0, buffer, 5), TILEDB_FS_ERR); CHECK_RC(gcs_instance->read_from_file("non-existent-dir/foo", 0, buffer, 5), TILEDB_FS_ERR); CHECK_RC(gcs_instance->close_file("non-existent-dir/foo"), TILEDB_FS_OK); free(buffer); } TEST_CASE_METHOD(GCSTestFixture, "Test GCS large read/write file", "[read-write-large]") { if (gcs_instance == nullptr) { return; } std::string test_dir("read_write_large"); // size_t size = ((size_t)TILEDB_UT_MAX_WRITE_COUNT)*4 size_t size = ((size_t)TILEDB_UT_MAX_WRITE_COUNT); void *buffer = malloc(size); if (buffer) { memset(buffer, 'B', size); REQUIRE(gcs_instance->create_dir(test_dir) == TILEDB_FS_OK); CHECK_RC(gcs_instance->write_to_file(test_dir+"/foo", buffer, size), TILEDB_FS_OK); CHECK_RC(gcs_instance->write_to_file(test_dir+"/foo", buffer, size), TILEDB_FS_OK); CHECK_RC(gcs_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK(gcs_instance->is_file(test_dir+"/foo")); CHECK((size_t)gcs_instance->file_size(test_dir+"/foo") == size*2); void *buffer1 = malloc(size); if (buffer1) { memset(buffer1, 0, size); CHECK_RC(gcs_instance->read_from_file(test_dir+"/foo", 0, buffer1, size), TILEDB_FS_OK); CHECK(memcmp(buffer, buffer1, size) == 0); free(buffer1); } free(buffer); } } TEST_CASE_METHOD(GCSTestFixture, "Test GCS operations", "[parallel]") { if (gcs_instance == nullptr) { return; } std::string test_dir("parallel"); REQUIRE(gcs_instance->create_dir(test_dir) == TILEDB_FS_OK); bool complete = true; uint iterations = 2; #pragma omp parallel for for (uint i=0; iwrite_to_file(filename, buffer, size), TILEDB_FS_OK); free(buffer); } else { complete = false; } } } CHECK_RC(gcs_instance->sync_path(test_dir), TILEDB_FS_OK); if (complete) { #pragma omp parallel for for (uint i=0; iclose_file(filename), TILEDB_FS_OK); CHECK(gcs_instance->is_file(filename)); CHECK(gcs_instance->file_size(filename) == (size_t)(10*1024*1024*2)); } } CHECK_RC(gcs_instance->delete_dir(test_dir), TILEDB_FS_OK); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/storage_manager/test_posixfs.cc000066400000000000000000000370041453617025200263160ustar00rootroot00000000000000/** * @file test_posixfs.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2018-2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the PosixFS class */ #include "catch.h" #include "storage_posixfs.h" #include "utils.h" #include #include #include #include #include #include #include #ifdef HAVE_OPENMP #include #endif #include class PosixFSTestFixture { protected: PosixFS fs; TempDir *td; std::string test_dir; PosixFSTestFixture() { td = new TempDir(); CHECK(fs.is_dir(td->get_temp_dir())); test_dir = td->get_temp_dir()+"/test_posixfs_dir"; if (getenv( "TILEDB_DISABLE_FILE_LOCKING")) { unsetenv( "TILEDB_DISABLE_FILE_LOCKING"); } REQUIRE(fs.locking_support()); CHECK(fs.locking_support()); } ~PosixFSTestFixture() { delete td; } }; TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS cwd", "[cwd]") { REQUIRE(fs.current_dir().length() > 0); REQUIRE(fs.is_dir(fs.current_dir())); REQUIRE(!fs.is_file(fs.current_dir())); REQUIRE(fs.real_dir(fs.current_dir()).length() > 0); } TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS real_dir", "[real_dir]") { CHECK(fs.real_dir("").compare(fs.current_dir()) == 0); CHECK(fs.real_dir(".").compare(fs.current_dir()) == 0); CHECK(fs.real_dir("./").compare(fs.current_dir()) == 0); CHECK(fs.real_dir("./foo").compare(fs.current_dir()+"/foo") == 0); CHECK(fs.real_dir("/").compare("/") == 0); CHECK(fs.real_dir("~").compare(getenv("HOME")) == 0); CHECK(fs.real_dir("~/foo").compare(std::string(getenv("HOME"))+"/foo") == 0); CHECK(fs.real_dir("/tmp").compare("/tmp") == 0); CHECK(fs.real_dir("/tmp/./xxx").compare("/tmp/xxx") == 0); CHECK(fs.real_dir("/tmp//xxx").compare("/tmp/xxx") == 0); CHECK(fs.real_dir("/tmp/../xxx").compare("/xxx") == 0); CHECK(fs.real_dir("xxx").compare(fs.current_dir()+"/xxx") == 0); } TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS dir", "[dir]") { test_dir += "dir"; CHECK_RC(fs.create_dir(test_dir), TILEDB_FS_OK); CHECK(fs.is_dir(test_dir)); CHECK(!fs.is_file(test_dir)); CHECK_RC(fs.create_dir(test_dir), TILEDB_FS_ERR); // Dir already exists CHECK_RC(fs.create_dir("/non-existent-dir/foo"), TILEDB_FS_ERR); CHECK_RC(fs.create_file(test_dir, O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_ERR); CHECK_RC(fs.create_file(fs.real_dir(test_dir), O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_ERR); CHECK(fs.file_size(test_dir) == TILEDB_FS_ERR); CHECK(fs.get_dirs(".").size() > 0); CHECK(fs.get_dirs(test_dir).size() == 0); CHECK(fs.get_dirs("non-existent-dir").size() == 0); std::string new_dir = test_dir+"-new"; CHECK_RC(fs.move_path(test_dir, new_dir), TILEDB_FS_OK); CHECK(fs.is_dir(new_dir)); CHECK(!fs.is_dir(test_dir)); CHECK_RC(fs.move_path(new_dir, test_dir), TILEDB_FS_OK); CHECK(!fs.is_dir(new_dir)); CHECK(fs.is_dir(test_dir)); CHECK_RC(fs.move_path("non-existent-dir", new_dir), TILEDB_FS_ERR); CHECK_RC(fs.move_path(test_dir, test_dir), TILEDB_FS_OK); CHECK(fs.is_dir(test_dir)); CHECK_RC(fs.sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(fs.sync_path("non-existent-dir"), TILEDB_FS_OK); CHECK_RC(fs.delete_dir(test_dir), TILEDB_FS_OK); CHECK_RC(fs.delete_dir("non-existent-dir"), TILEDB_FS_ERR); } TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS file", "[file]") { test_dir += "file"; CHECK_RC(fs.create_dir(test_dir), 0); REQUIRE(fs.is_dir(test_dir)); CHECK_RC(fs.create_file(test_dir+"/foo", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(fs.is_file(test_dir+"/foo")); CHECK(!fs.is_dir(test_dir+"/foo")); CHECK(fs.file_size(test_dir+"/foo") == 0); CHECK(fs.file_size(test_dir+"/foo1") == TILEDB_FS_ERR); CHECK(fs.get_files(".").size() > 0); CHECK(fs.get_files(test_dir).size() == 1); CHECK(fs.get_files("non-existent-dir").size() == 0); CHECK_RC(fs.move_path(test_dir+"/foo", test_dir+"/foo2"), TILEDB_FS_OK); CHECK(fs.is_file(test_dir+"/foo2")); CHECK(!fs.is_file(test_dir+"/foo")); CHECK_RC(fs.move_path(test_dir+"/foo2", test_dir+"/foo"), TILEDB_FS_OK); CHECK(!fs.is_file(test_dir+"/foo2")); CHECK(fs.is_file(test_dir+"/foo")); CHECK_RC(fs.move_path(test_dir+"/foo", test_dir+"/foo"), TILEDB_FS_OK); CHECK(fs.is_file(test_dir+"/foo")); CHECK_RC(fs.sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs.sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(fs.delete_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs.delete_file(test_dir+"/foo1"), TILEDB_FS_ERR); } TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS read/write file", "[read-write]") { test_dir += "read_write"; CHECK_RC(fs.create_dir(test_dir), TILEDB_FS_OK); REQUIRE(fs.is_dir(test_dir)); CHECK_RC(fs.write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); REQUIRE(fs.is_file(test_dir+"/foo")); CHECK(fs.file_size(test_dir+"/foo") == 5); void *buffer = malloc(20); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer, 0), TILEDB_FS_OK); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer, 2), TILEDB_FS_OK); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer, 5), TILEDB_FS_OK); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer, 6), TILEDB_FS_ERR); CHECK_RC(fs.read_from_file(test_dir+"/foo", 3, buffer, 2), TILEDB_FS_OK); CHECK_RC(fs.read_from_file(test_dir+"/foo", 3, buffer, 6), TILEDB_FS_ERR); CHECK_RC(fs.read_from_file(test_dir+"/foo", 6, buffer, 2), TILEDB_FS_ERR); CHECK_RC(fs.write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); CHECK(fs.file_size(test_dir+"/foo") == 10); CHECK_RC(fs.close_file(test_dir+"/foo"), TILEDB_FS_OK); // NOP when there is no locking support CHECK_RC(fs.sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs.delete_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs.sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(fs.read_from_file("non-existent-dir/foo", 0, buffer, 5), TILEDB_FS_ERR); CHECK_RC(fs.write_to_file("non-existent-dir/foo", "hello", 5), TILEDB_FS_ERR); CHECK_RC(fs.close_file("non-existent-dir/foo"), TILEDB_FS_OK); // NOP when there is no locking support free(buffer); } TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS large read/write file", "[read-write-large]") { if (!is_env_set("TILEDB_TEST_POSIXFS_LARGE")) { return; } test_dir += "read_write_large"; ssize_t size = (ssize_t)TILEDB_UT_MAX_WRITE_COUNT*4; void *buffer = malloc(size); if (buffer) { memset(buffer, 'B', size); REQUIRE(fs.create_dir(test_dir) == TILEDB_FS_OK); CHECK_RC(fs.write_to_file(test_dir+"/foo", buffer, size), TILEDB_FS_OK); CHECK(fs.file_size(test_dir+"/foo") == size); void *buffer1 = malloc(size); if (buffer1) { memset(buffer1, 0, size); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer1, size), TILEDB_FS_OK); CHECK(memcmp(buffer, buffer1, size) == 0); free(buffer1); } free(buffer); } } TEST_CASE_METHOD(PosixFSTestFixture, "Test PosixFS parallel operations", "[parallel]") { test_dir += "parallel"; REQUIRE(fs.create_dir(test_dir) == TILEDB_FS_OK); bool complete = true; uint iterations = 4; #pragma omp parallel for for (uint i=0; i(disable_file_locking_env.c_str())) == 0); const char *value = disable_file_locking_value.c_str(); const char *env_value = getenv("TILEDB_DISABLE_FILE_LOCKING"); REQUIRE(env_value != NULL); CHECK(strcmp(value, env_value) == 0); PosixFS fs; if (strcasecmp(value, "true") == 0 || strcmp(value, "1") == 0) { CHECK(!fs.locking_support()); } else { CHECK(fs.locking_support()); } } TEST_CASE("Test locking support", "[locking_support]") { PosixFS fs; CHECK(!fs.disable_file_locking()); // default CHECK(fs.locking_support()); fs.set_disable_file_locking(true); CHECK(fs.disable_file_locking()); CHECK(!fs.locking_support()); fs.set_disable_file_locking(false); CHECK(!fs.disable_file_locking()); CHECK(fs.locking_support()); test_locking_support("True"); test_locking_support("true"); test_locking_support("TRUE"); test_locking_support("1"); test_locking_support("0"); test_locking_support("False"); test_locking_support("false"); test_locking_support("FALSE"); test_locking_support("Gibberish"); } void test_keep_file_handles_open_support(const std::string& keep_file_handles_open_value) { std::string keep_file_handles_open_env = "TILEDB_KEEP_FILE_HANDLES_OPEN="+keep_file_handles_open_value; CHECK(putenv(const_cast(keep_file_handles_open_env.c_str())) == 0); const char *value = keep_file_handles_open_value.c_str(); const char *env_value = getenv("TILEDB_KEEP_FILE_HANDLES_OPEN"); REQUIRE(env_value != NULL); CHECK(strcmp(value, env_value) == 0); PosixFS fs; if (strcasecmp(value, "true") == 0 || strcmp(value, "1") == 0) { CHECK(fs.keep_write_file_handles_open()); } else { CHECK(!fs.keep_write_file_handles_open()); } } TEST_CASE("Test keep file handles open", "[keep_file_handles_open_support]") { PosixFS fs; CHECK(!fs.keep_write_file_handles_open()); // default fs.set_keep_write_file_handles_open(true); CHECK(fs.keep_write_file_handles_open()); fs.set_keep_write_file_handles_open(false); CHECK(!fs.keep_write_file_handles_open()); test_keep_file_handles_open_support("True"); test_keep_file_handles_open_support("true"); test_keep_file_handles_open_support("TRUE"); test_keep_file_handles_open_support("1"); test_keep_file_handles_open_support("0"); test_keep_file_handles_open_support("False"); test_keep_file_handles_open_support("false"); test_keep_file_handles_open_support("FALSE"); test_keep_file_handles_open_support("Gibberish"); } TEST_CASE("Test writing with keeps file descriptors open until explicitly closed", "[write_keep_file_handles_open]") { CHECK(setenv("TILEDB_KEEP_FILE_HANDLES_OPEN", "1", 1) == 0); PosixFS fs; REQUIRE(fs.keep_write_file_handles_open()); std::string test_dir = "test_posixfs_dir_locking"; if (fs.is_dir(test_dir)) { REQUIRE(fs.delete_dir(test_dir) == TILEDB_OK); } CHECK_RC(fs.create_dir(test_dir), 0); REQUIRE(fs.is_dir(test_dir)); CHECK_RC(fs.write_to_file(test_dir+"/foo", "hello", 6), TILEDB_FS_OK); REQUIRE(fs.is_file(test_dir+"/foo")); CHECK(fs.file_size(test_dir+"/foo") == 6); void *buffer = malloc(20); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer, 6), TILEDB_FS_ERR); // No simultaneous read/write allowed on open files CHECK_RC(fs.sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs.close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs.read_from_file(test_dir+"/foo", 0, buffer, 6), TILEDB_FS_OK); char *buffer_str = (char *)buffer; CHECK(strlen(buffer_str) == 5); CHECK(strcmp(buffer_str, "hello") == 0); CHECK_RC(fs.delete_file(test_dir+"/foo"), TILEDB_FS_OK); REQUIRE(!fs.is_file(test_dir+"/foo")); // Test multiple writes int n_iter = 2; for (int i=0; iwrite_to_file(test_dir+"/foo_sync", "hello", 6), TILEDB_FS_OK); REQUIRE(fs_sync->is_file(test_dir+"/foo_sync")); CHECK(fs_sync->file_size(test_dir+"/foo_sync") == 6); CHECK_RC(fs_sync->sync_path(test_dir+"/foo"), TILEDB_FS_OK); delete fs_sync; // Perform some writes and do not close/sync the file. This should log an error but complete successfully PosixFS *fs_no_close = new PosixFS(); CHECK_RC(fs_no_close->write_to_file(test_dir+"/foo_no_close", "hello", 6), TILEDB_FS_OK); REQUIRE(fs_no_close->is_file(test_dir+"/foo_no_close")); CHECK(fs_no_close->file_size(test_dir+"/foo_no_close") == 6); delete fs_no_close; unsetenv("TILEDB_KEEP_FILE_HANDLES_OPEN"); PosixFS fs1; void *buffer = malloc(6*n_iter); CHECK_RC(fs1.read_from_file(test_dir+"/foo", 0, buffer, 6*n_iter), TILEDB_FS_OK); CHECK(fs1.file_size(test_dir+"/foo") == 6*n_iter); CHECK_RC(fs1.delete_file(test_dir+"/foo"), TILEDB_FS_OK); // Try closing non-existent file REQUIRE(!fs1.is_file(test_dir+"/foo")); CHECK_RC(fs1.close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(fs1.delete_dir(test_dir), TILEDB_FS_OK); free(buffer); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/storage_manager/test_s3_storage.cc000066400000000000000000000257271453617025200267050ustar00rootroot00000000000000/** * @file test_s3_storage.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2021 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the S3 storage class */ #include "catch.h" #include "storage_s3.h" #include "uri.h" #include "utils.h" #include #include #include #include class S3TestFixture { protected: TempDir *temp_dir = NULL; S3 *s3_instance = NULL; bool is_minio = false; S3TestFixture() { if (is_s3_storage_path(get_test_dir())) { try { temp_dir = new TempDir(); std::string home_dir = temp_dir->get_temp_dir()+"/test_s3"; s3_instance = new S3(home_dir); CHECK(!s3_instance->locking_support()); if (getenv("MINIO_ACCESS_KEY")) { is_minio = true; } } catch(...) { INFO("S3 Storage could not be credentialed"); } } INFO("S3 Storage not specified as a --test-dir option"); } ~S3TestFixture() { if (s3_instance) { delete s3_instance; } if (temp_dir) { delete temp_dir; } } }; TEST_CASE("Test S3 constructor", "[constr]") { CHECK_THROWS(new S3("s3kdd://my_container/path")); CHECK_THROWS(new S3("az://my_container@my_account.blob.core.windows.net/path")); CHECK_THROWS(new S3("s3://my_container/path")); } TEST_CASE_METHOD(S3TestFixture, "Test S3 cwd", "[cwd]") { if (s3_instance == nullptr) { return; } REQUIRE(s3_instance->current_dir().length() > 0); REQUIRE(s3_instance->create_dir(s3_instance->current_dir()) == TILEDB_FS_OK); REQUIRE(s3_instance->is_dir(s3_instance->current_dir())); REQUIRE(!s3_instance->is_file(s3_instance->current_dir())); REQUIRE(s3_instance->real_dir(s3_instance->current_dir()).length() > 0); } TEST_CASE_METHOD(S3TestFixture, "Test S3 real_dir", "[real_dir]") { if (s3_instance == nullptr) { return; } CHECK(s3_instance->real_dir("").compare(s3_instance->current_dir()) == 0); CHECK(s3_instance->real_dir("xxx").compare(s3_instance->current_dir()+"/xxx") == 0); CHECK(s3_instance->real_dir("xxx/yyy").compare(s3_instance->current_dir()+"/xxx/yyy") == 0); CHECK(s3_instance->real_dir("/xxx/yyy").compare("xxx/yyy") == 0); s3_uri test_uri(get_test_dir()); CHECK(s3_instance->real_dir(get_test_dir()).compare(test_uri.path().substr(1)) == 0); CHECK_THROWS(s3_instance->real_dir("xxx://yyy")); } TEST_CASE_METHOD(S3TestFixture, "Test S3 dir", "[dir]") { if (s3_instance == nullptr) { return; } CHECK(!s3_instance->is_dir("non-existent-dir")); CHECK(!s3_instance->is_dir("non-existent-parent-dir/dir")); std::string test_dir("dir"); CHECK_RC(s3_instance->create_dir(test_dir), TILEDB_FS_OK); CHECK(s3_instance->is_dir(test_dir)); CHECK(!s3_instance->is_file(test_dir)); CHECK_RC(s3_instance->create_dir(test_dir), TILEDB_FS_ERR); // Dir already exists CHECK_RC(s3_instance->create_file(test_dir, 0, 0), TILEDB_FS_ERR); CHECK_RC(s3_instance->create_file(s3_instance->real_dir(test_dir), 0, 0), TILEDB_FS_ERR); CHECK(s3_instance->file_size(test_dir) == TILEDB_FS_ERR); CHECK(s3_instance->get_dirs(test_dir).size() == 0); CHECK(s3_instance->get_files(test_dir).size() == 0); CHECK(s3_instance->get_dirs("non-existent-dir").size() == 0); CHECK(s3_instance->get_files("non-existent-dir").size() == 0); // TBD: move_path std::string new_dir = test_dir+"-new"; CHECK_THROWS(s3_instance->move_path(test_dir, new_dir)); CHECK_RC(s3_instance->sync_path(test_dir), TILEDB_FS_OK); // No support for returning errors for non-existent paths CHECK_RC(s3_instance->sync_path("non-existent-dir"), TILEDB_FS_OK); CHECK_RC(s3_instance->delete_dir(test_dir), TILEDB_FS_OK); CHECK_RC(s3_instance->delete_dir("non-existent-dir"), TILEDB_FS_ERR); CHECK(!s3_instance->is_dir(test_dir)); CHECK(!s3_instance->is_file(test_dir)); } TEST_CASE_METHOD(S3TestFixture, "Test S3 file", "[file]") { if (s3_instance == nullptr) { return; } std::string test_dir("file"); CHECK_RC(s3_instance->create_dir(test_dir), 0); REQUIRE(s3_instance->is_dir(test_dir)); CHECK_RC(s3_instance->create_file(test_dir+"/foo", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(s3_instance->is_file(test_dir+"/foo")); CHECK(!s3_instance->is_dir(test_dir+"/foo")); CHECK(s3_instance->file_size(test_dir+"/foo") == 0); CHECK(s3_instance->file_size(test_dir+"/foo1") == TILEDB_FS_ERR); CHECK(s3_instance->get_files(test_dir).size() == 1); CHECK(s3_instance->get_files("non-existent-dir").size() == 0); CHECK_RC(s3_instance->create_file(test_dir+"/foo1", O_WRONLY|O_CREAT, S_IRWXU), TILEDB_FS_OK); CHECK(s3_instance->get_files(test_dir).size() == 2); CHECK_RC(s3_instance->sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(s3_instance->sync_path(test_dir), TILEDB_FS_OK); CHECK_RC(s3_instance->delete_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(s3_instance->delete_file(test_dir+"/foo1"), TILEDB_FS_OK); CHECK_RC(s3_instance->delete_file(test_dir+"/foo2"), TILEDB_FS_ERR); CHECK(!s3_instance->is_file(test_dir+"/foo1")); CHECK(!s3_instance->is_file(test_dir+"/foo2")); CHECK(!s3_instance->is_dir(test_dir+"/foo1")); CHECK(!s3_instance->is_dir(test_dir+"/foo2")); } TEST_CASE_METHOD(S3TestFixture, "Test S3 read/write file", "[read-write]") { if (s3_instance == nullptr) { return; } std::string test_dir("read_write"); CHECK_RC(s3_instance->create_dir(test_dir), TILEDB_FS_OK); REQUIRE(s3_instance->is_dir(test_dir)); CHECK_RC(s3_instance->write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); CHECK_RC(s3_instance->sync_path(test_dir+"/foo"), TILEDB_FS_OK); CHECK_RC(s3_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); auto files = s3_instance->get_files(test_dir); CHECK(files.size() == 1); REQUIRE(s3_instance->is_file(test_dir+"/foo")); CHECK(s3_instance->file_size(test_dir+"/foo") == 5); void *buffer = malloc(20); memset(buffer, 'X', 20); CHECK_RC(s3_instance->read_from_file(test_dir+"/foo", 0, buffer, 0), TILEDB_FS_OK); CHECK_RC(s3_instance->read_from_file(test_dir+"/foo", 0, buffer, 2), TILEDB_FS_OK); CHECK(((char *)buffer)[0] == 'h'); CHECK(((char *)buffer)[1] == 'e'); CHECK_RC(s3_instance->read_from_file(test_dir+"/foo", 0, buffer, 5), TILEDB_FS_OK); CHECK(((char *)buffer)[4] == 'o'); CHECK_RC(s3_instance->read_from_file(test_dir+"/foo", 0, buffer, 6), TILEDB_FS_ERR); CHECK_RC(s3_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); if (!is_minio) { memset(buffer, 'X', 20); CHECK_RC(s3_instance->write_to_file(test_dir+"/foo1", buffer, 20), TILEDB_FS_OK); CHECK_RC(s3_instance->close_file(test_dir+"/foo1"), TILEDB_FS_OK); REQUIRE(s3_instance->is_file(test_dir+"/foo1")); for (int i=0; i<4; i++) { void *buf = malloc(5); memset(buf, 'Z', 5); CHECK_RC(s3_instance->read_from_file(test_dir+"/foo1", i*5, buf, 5), TILEDB_FS_OK); for (int j=0; j<5; j++) { CHECK(((char *)buf)[j] == 'X'); } free(buf); } } // This part is different from Azure Blob Storage where you are allowed to write as many chunks in any size if (!is_minio) { // No overwriting allowed with minio, so skip this set of tests for now CHECK_RC(s3_instance->write_to_file(test_dir+"/foo", "hello", 5), TILEDB_FS_OK); CHECK_RC(s3_instance->write_to_file(test_dir+"/foo", " there ", 6), TILEDB_FS_ERR); // Prev part should at least be 5M, otherwise should be aborted CHECK_RC(s3_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK(s3_instance->file_size(test_dir+"/foo") == 5); // Previous file size as the current set of writes failed } CHECK_RC(s3_instance->read_from_file(test_dir+"/non-existent-file", 0, buffer, 5), TILEDB_FS_ERR); CHECK_RC(s3_instance->read_from_file("non-existent-dir/foo", 0, buffer, 5), TILEDB_FS_ERR); CHECK_RC(s3_instance->close_file("non-existent-dir/foo"), TILEDB_FS_OK); free(buffer); } TEST_CASE_METHOD(S3TestFixture, "Test S3 large read/write file", "[read-write-large]") { if (s3_instance == nullptr || is_minio) { return; } std::string test_dir("read_write_large"); // size_t size = ((size_t)TILEDB_UT_MAX_WRITE_COUNT)*4 size_t size = ((size_t)TILEDB_UT_MAX_WRITE_COUNT); void *buffer = malloc(size); if (buffer) { memset(buffer, 'B', size); REQUIRE(s3_instance->create_dir(test_dir) == TILEDB_FS_OK); CHECK_RC(s3_instance->write_to_file(test_dir+"/foo", buffer, size), TILEDB_FS_OK); CHECK_RC(s3_instance->close_file(test_dir+"/foo"), TILEDB_FS_OK); CHECK(s3_instance->is_file(test_dir+"/foo")); CHECK((size_t)s3_instance->file_size(test_dir+"/foo") == size); void *buffer1 = malloc(size); if (buffer1) { memset(buffer1, 0, size); CHECK_RC(s3_instance->read_from_file(test_dir+"/foo", 0, buffer1, size), TILEDB_FS_OK); CHECK(memcmp(buffer, buffer1, size) == 0); free(buffer1); } free(buffer); } } TEST_CASE_METHOD(S3TestFixture, "Test S3 operations", "[parallel]") { if (s3_instance == nullptr) { return; } std::string test_dir("parallel"); REQUIRE(s3_instance->create_dir(test_dir) == TILEDB_FS_OK); bool complete = true; uint iterations = 2; size_t size = 10*1024*1024; #pragma omp parallel for for (uint i=0; iwrite_to_file(filename, buffer, size), TILEDB_FS_OK); free(buffer); } else { complete = false; } } } CHECK_RC(s3_instance->close_file(test_dir), TILEDB_FS_OK); if (complete) { #pragma omp parallel for for (uint i=0; iclose_file(filename), TILEDB_FS_OK); sleep(10); CHECK(s3_instance->is_file(filename)); CHECK((size_t)s3_instance->file_size(filename) == size*2); } } CHECK_RC(s3_instance->delete_dir(test_dir), TILEDB_FS_OK); } genomicsdb-0.0~git20231212.9d7ddd0/test/src/storage_manager/test_storage_buffer.cc000066400000000000000000000256501453617025200276240ustar00rootroot00000000000000/** * @file test_storage_buffer.cc * * @section LICENSE * * The MIT License * * @copyright Copyright (c) 2021-2022 Omics Data Automation, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @section DESCRIPTION * * Tests for the StorageBuffer class */ #include "catch.h" #include "storage_azure_blob.h" #include "storage_buffer.h" #include "storage_gcs.h" #include "storage_posixfs.h" #include "storage_s3.h" #include "utils.h" TEST_CASE_METHOD(TempDir, "Test Storage Buffer Basic", "[basic]") { if (get_temp_dir().find("://") != std::string::npos) { return; } std::string filename = get_temp_dir()+"/test-file"; std::vector buffer(10); std::generate(buffer.begin(), buffer.end(), std::rand); size_t chunk_size = 0; PosixFS fs; StorageBuffer storage_buffer(&fs, filename, chunk_size); CHECK_RC(storage_buffer.append_buffer(buffer.data(), 10), TILEDB_BF_ERR); // No upload buffer size set chunk_size = 100; StorageBuffer storage_buffer1(&fs, filename, chunk_size); CHECK_RC(storage_buffer1.append_buffer(buffer.data(), 10), TILEDB_BF_OK); CHECK_RC(storage_buffer1.finalize(), TILEDB_BF_OK); std::vector check_buffer(10); CHECK(fs.file_size(filename) == 10); CHECK_RC(fs.read_from_file(filename, 0, check_buffer.data(), 10), TILEDB_FS_OK); CHECK(memcmp(buffer.data(), check_buffer.data(), 10) == 0); std::vector read_buffer(10); StorageBuffer storage_buffer2(&fs, filename+"-non-existent", chunk_size, /*is_read*/true); CHECK_RC(storage_buffer2.read_buffer(0, read_buffer.data(), 10), TILEDB_BF_ERR); // Non existent file StorageBuffer storage_buffer3(&fs, filename, 0, /*is_read*/true); CHECK_RC(storage_buffer3.read_buffer(0, read_buffer.data(), 10), TILEDB_BF_ERR); // No download buffer size chunk_size = 100; StorageBuffer storage_buffer4(&fs, filename, chunk_size, /*is_read*/true); CHECK_RC(storage_buffer4.read_buffer(0, read_buffer.data(), 10), TILEDB_BF_OK); CHECK(memcmp(buffer.data(), read_buffer.data(), 10) == 0); CHECK_RC(storage_buffer4.read_buffer(0, read_buffer.data(), 100), TILEDB_BF_ERR); // Reading past file } int write_to_file_after_compression(StorageFS *fs, const std::string& filename, const char *str, size_t size, int compression_type) { size_t chunk_size = 1024; CompressedStorageBuffer *buffer = new CompressedStorageBuffer(fs, filename, chunk_size, false, compression_type); if (buffer->append_buffer(str, size)) { delete buffer; return TILEDB_ERR; } if (buffer->finalize()) { delete buffer; return TILEDB_ERR; } return TILEDB_OK; } int read_from_file_after_decompression(StorageFS *fs, const std::string& filename, void *bytes, size_t length, int compression_type) { size_t chunk_size = 1024; CompressedStorageBuffer *buffer = new CompressedStorageBuffer(fs, filename, chunk_size, true, compression_type); if (buffer->read_buffer(bytes, length)) { delete buffer; return TILEDB_ERR; } int rc = buffer->finalize(); delete buffer; return rc; } TEST_CASE_METHOD(TempDir, "Test Storage Buffer with compression", "[compr]") { if (get_temp_dir().find("://") != std::string::npos) { return; } PosixFS test_fs; PosixFS *fs = &test_fs; char *buffer = NULL; size_t buffer_length = 0; CHECK(write_to_file_after_compression(fs, "/non-existent-file", "Hello", 6, TILEDB_NO_COMPRESSION) == TILEDB_ERR); CHECK(write_to_file_after_compression(fs, "/non-existent-file", "Hello", 6, TILEDB_GZIP) == TILEDB_ERR); // OK since buffer and/or buffer_length is 0 and nothing to do CHECK(read_from_file_after_decompression(fs, "/non-existent-file", buffer, 30, TILEDB_NO_COMPRESSION) == TILEDB_OK); CHECK(read_from_file_after_decompression(fs, "/non-existent-file", buffer, buffer_length, TILEDB_GZIP) == TILEDB_OK); buffer=(char *)malloc(10); buffer_length = 6; // length of "Hello" CHECK(read_from_file_after_decompression(fs, "/non-existent-file", buffer, 30, TILEDB_NO_COMPRESSION) == TILEDB_ERR); CHECK(read_from_file_after_decompression(fs, "/non-existent-file", buffer, buffer_length, TILEDB_GZIP) == TILEDB_ERR); const std::string compressed_file = get_temp_dir()+"/compressed_foo"; CHECK(write_to_file_after_compression(fs, "/non-existent-file", "Hello", 6, 1000) == TILEDB_ERR); // Unsupported type REQUIRE(!write_to_file_after_compression(fs, compressed_file, "Hello", 6, TILEDB_GZIP)); CHECK(read_from_file_after_decompression(fs, compressed_file, buffer, buffer_length, 1000) == TILEDB_ERR); // Unsupported type CHECK(!read_from_file_after_decompression(fs, compressed_file, buffer, buffer_length, TILEDB_GZIP)); CHECK(buffer_length == 6); CHECK(!strcmp(buffer, "Hello")); CHECK(read_from_file_after_decompression(fs, compressed_file, buffer, 10, TILEDB_GZIP) == TILEDB_ERR); // Reading past decompressed filesize free(buffer); } class TestBufferedWrite : public TempDir { public: void write(const std::string& filename, StorageBuffer *storage_buffer, char *buffer, size_t size) { size_t unprocessed = size; do { size_t bytes_to_process = unprocessed<1024?unprocessed:std::rand()%1024; CHECK_RC(storage_buffer->append_buffer(buffer+size-unprocessed, bytes_to_process), TILEDB_BF_OK); unprocessed -= bytes_to_process; } while(unprocessed); CHECK_RC(storage_buffer->finalize(), TILEDB_BF_OK); } void read(const std::string& filename, StorageBuffer *storage_buffer, char *buffer, size_t size) { size_t unprocessed = size; do { CHECK(size-unprocessed >= 0); size_t bytes_to_process = unprocessed<1024?unprocessed:std::rand()%1024; CHECK_RC(storage_buffer->read_buffer(size-unprocessed, buffer+size-unprocessed, bytes_to_process), TILEDB_BF_OK); unprocessed -= bytes_to_process; } while(unprocessed > 0); CHECK_RC(storage_buffer->finalize(), TILEDB_BF_OK); } void read_with_implicit_offset(const std::string& filename, CompressedStorageBuffer *storage_buffer, char *buffer, size_t size) { size_t unprocessed = size; do { CHECK(size-unprocessed >= 0); size_t bytes_to_process = unprocessed<1024?unprocessed:std::rand()%1024; CHECK_RC(storage_buffer->read_buffer(buffer+size-unprocessed, bytes_to_process), TILEDB_BF_OK); unprocessed -= bytes_to_process; } while(unprocessed > 0); CHECK_RC(storage_buffer->finalize(), TILEDB_BF_OK); } }; TEST_CASE_METHOD(TestBufferedWrite, "Test Storage Buffer with buffered reading/writing of large files", "[large-file]") { size_t size = 1024*1024*20 + 1024; // 20M + 1024 std::vector buffer(size); std::generate(buffer.begin(), buffer.end(), std::rand); std::shared_ptr fs; bool is_posix = false; // Buffered write std::string filename = get_temp_dir()+"/buffered_file"; if (is_gcs_path(filename)) { fs = std::make_shared(filename); } else if (is_azure_blob_storage_path(filename)) { fs = std::make_shared(filename); } else if (is_s3_storage_path(filename)) { fs = std::make_shared(filename); } else { fs = std::make_shared(); is_posix = true; } auto write_chunk_size = fs->get_upload_buffer_size()?fs->get_upload_buffer_size():10240; // default chunk auto read_chunk_size = fs->get_download_buffer_size()?fs->get_download_buffer_size():5120; // default chunk // Buffered write StorageBuffer storage_buffer(fs.get(), filename, write_chunk_size, /*is_read*/false); write(filename, &storage_buffer, buffer.data(), size); CHECK((size_t)fs->file_size(filename) == size); // Check buffered write results with unbuffered read std::vector read_buffer(size); CHECK_RC(fs->read_from_file(filename, 0, read_buffer.data(), size), TILEDB_BF_OK); CHECK_RC(fs->close_file(filename), TILEDB_BF_OK); CHECK(memcmp(buffer.data(), read_buffer.data(), size) == 0); // Buffered read StorageBuffer read_storage_buffer(fs.get(), filename, read_chunk_size, /*is_read*/true); memset(read_buffer.data(), 0, size); read(filename, &read_storage_buffer, read_buffer.data(), size); CHECK(memcmp(buffer.data(), read_buffer.data(), size) == 0); // Buffered read with implicit offset CompressedStorageBuffer read_storage_buffer_1(fs.get(), filename, write_chunk_size, /*is_read*/true); memset(read_buffer.data(), 0, size); read_with_implicit_offset(filename, &read_storage_buffer_1, read_buffer.data(), size); CHECK(memcmp(buffer.data(), read_buffer.data(), size) == 0); // Buffered write with compression filename += ".compress"; CompressedStorageBuffer storage_buffer_with_compression(fs.get(), filename, write_chunk_size, false, TILEDB_GZIP, TILEDB_COMPRESSION_LEVEL_GZIP); write(filename, &storage_buffer_with_compression, buffer.data(), size); // Buffered read with decompression CompressedStorageBuffer read_storage_buffer_with_compression(fs.get(), filename, read_chunk_size, true, TILEDB_GZIP); memset(read_buffer.data(), 0, size); read_with_implicit_offset(filename, &read_storage_buffer_with_compression, read_buffer.data(), size); CHECK(memcmp(buffer.data(), read_buffer.data(), size) == 0); // Buffered write with compression and buffer set to all one's filename += ".simple"; memset(buffer.data(), 'R', size); if (is_posix) { // This will test out all paths to write out compressed bytes even for Posix fs->set_upload_buffer_size(write_chunk_size*2); } CompressedStorageBuffer simple_storage_buffer_with_compression(fs.get(), filename, write_chunk_size, false, TILEDB_GZIP, TILEDB_COMPRESSION_LEVEL_GZIP); write(filename, &simple_storage_buffer_with_compression, buffer.data(), size); // Buffer read with decompression for buffer set to one's CompressedStorageBuffer read_simple_storage_buffer_with_compression(fs.get(), filename, read_chunk_size, true, TILEDB_GZIP); memset(read_buffer.data(), 0, size); read_with_implicit_offset(filename, &read_simple_storage_buffer_with_compression, read_buffer.data(), size); CHECK(memcmp(buffer.data(), read_buffer.data(), size) == 0); }